get logos in main.py

Signed-off-by: Niv Sardi <[email protected]>

python/imtool.py

@@ -99,6 +99,7 @@ def mix(a, b, fx, fy):
     x = math.floor(fx*(aw - bw))
     y = math.floor(fy*(ah - bh))
 
+    # handle transparency
     mat = a[y:y+bh,x:x+bw]
     cols = b[:, :, :3]
     alpha = b[:, :, 3]/255

python/main.py

@@ -5,6 +5,7 @@ import shutil
 from bs4 import BeautifulSoup
 from progress.bar import ChargingBar
 
+import web
 from entity import Entity
 from common import selectors, defaults, mkdir
 
@@ -28,6 +29,8 @@ with open(f'{defaults.MAIN_CSV_PATH}.tmp', 'w', newline='') as csvfile:
         try:
             img = soup.select_one(selectors.logosbancos).attrs['src']
             img = img.replace('../', 'https://www.bcra.gob.ar/')
+            fn = f"{defaults.LOGOS_DATA_PATH}/{bco}.0.png"
+            web.get_img_logo(img, fn)
         except AttributeError as err:
             print('img', name, err)
             img = None

python/vendor.py

@@ -10,17 +10,18 @@ from common import defaults,mkdir
 import screenshot
 import web
 
+PARALLEL = 20
+
 def query_vendor_site(e: Entity):
-    page = web.get_page(e)
     fn = web.get_cert(e)
-    lfn = web.get_logos(e, page)
-    screenshot.sc_entity(e)
-    return (fn, lfn)
+    lfn = web.get_logos(e)
+    sfn = screenshot.sc_entity(e)
+    return (fn, lfn, sfn)
 
-def from_csv(fn):
+def from_csv(fn: str):
     with open(fn, newline='') as csvfile:
         reader = csv.DictReader(csvfile)
-        with concurrent.futures.ThreadPoolExecutor(max_workers = 5) as executor:
+        with concurrent.futures.ThreadPoolExecutor(max_workers = PARALLEL) as executor:
             futures = {executor.submit(query_vendor_site, e): e for e in [Entity.from_dict(d) for d in reader]}
             bar = ChargingBar('Processing', max=len(futures))
             for f in concurrent.futures.as_completed(futures):

python/web.py

@@ -34,7 +34,8 @@ def get_img_logo(src: str, fn):
             shutil.copyfileobj(res.raw, f)
         return fn
 
-def get_logos(e: Entity, page):
+def get_logos(e: Entity):
+    page = get_page(e)
     soup = BeautifulSoup(page.content, "html.parser")
     logos = soup.select(selectors.img_logo)
     logos.extend(soup.select(selectors.id_logo))
@@ -42,7 +43,7 @@ def get_logos(e: Entity, page):
 
     mkdir.make_dirs([defaults.LOGOS_DATA_PATH])
 
-    i = 0
+    i = 1
     lfn = []
     for l in logos:
         if 'src' in l.attrs: