defaults defaults defaults (and types)

Signed-off-by: Niv Sardi <[email protected]>

python/common/defaults.py

@@ -1 +1,16 @@
 DATA_PATH='./data'
+
+LABELS_PATH=f'{DATA_PATH}/labels'
+IMAGES_PATH=f'{DATA_PATH}/images'
+CERTS_PATH=f'{DATA_PATH}/certs'
+
+SQUARES_DATA_PATH=f'{DATA_PATH}/squares'
+SQUARES_LABELS_PATH=f'{SQUARES_DATA_PATH}/labels'
+SQUARES_IMAGES_PATH=f'{SQUARES_DATA_PATH}/images'
+
+DEBUG_PATH=f'{DATA_PATH}/debug'
+DEBUG_SQUARES_PATH=f'{DEBUG_PATH}/squares'
+
+LOGOS_DATA_PATH=f'{DATA_PATH}/logos'
+
+MAIN_CSV_PATH=f'{DATA_PATH}/entities.csv'

python/common/mkdir.py

@@ -0,0 +1,6 @@
+import pathlib
+
+def make_dirs(dirs: [str]):
+    for p in dirs:
+        pathlib.Path(p).mkdir(parents=True, exist_ok=True)
+

python/entity.py

@@ -2,8 +2,10 @@
 import csv
 from typing import NamedTuple
 
+from common import defaults
+
 def read_entities(fn):
-    with open('./data/entidades.csv', newline='') as csvfile:
+    with open(defaults.MAIN_DATA_PATH, newline='') as csvfile:
         reader = csv.DictReader(csvfile)
         bcos = { d['bco']:update(d, {'id': i}) for i, d in enumerate(reader)}
     return bcos

python/imtool.py

@@ -3,10 +3,10 @@
 import os
 import math
 import cv2
-import pathlib
 from typing import NamedTuple
 
 from entity import Entity
+from common import mkdir
 
 TILE_SIZE = 416
 TILE_OVERLAP = 0.8
@@ -69,9 +69,7 @@ def crop(id, fn, logos):
     img_out = f"./data/squares/images"
     txt_out = f"./data/squares/labels"
     debug_out = f"./data/debug"
-    pathlib.Path(debug_out).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(img_out).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(txt_out).mkdir(parents=True, exist_ok=True)
+    mkdir.make_dirs[debug_out, img_out, txt_out]
 
     im = cv2.imread(fn)
     rim = cv2.imread(fn)

python/main.py

@@ -1,5 +1,4 @@
 import csv
-import pathlib
 import requests
 import shutil
 
@@ -7,18 +6,16 @@ from bs4 import BeautifulSoup
 from progress.bar import ChargingBar
 
 from entity import Entity
-from common import selectors
-from common import defaults
+from common import selectors, defaults, mkdir
 
-pathlib.Path(f'{defaults.DATA_PATH}/logos').mkdir(parents=True, exist_ok=True)
-
-DATA_FILE = './data/entidades.csv'
 URL = 'http://www.bcra.gob.ar/SistemasFinancierosYdePagos/Entidades_financieras.asp'
 page = requests.get(URL)
 soup = BeautifulSoup(page.content, 'html.parser')
 
 options = soup.find(class_='form-control').find_all('option')
-with open(f'{DATA_FILE}.tmp', 'w', newline='') as csvfile:
+mkdir.make_dirs([defaults.DATA_PATH])
+
+with open(f'{defaults.MAIN_CSV_PATH}.tmp', 'w', newline='') as csvfile:
     writer = csv.writer(csvfile)
     writer.writerow(Entity.row_names())
 
@@ -46,11 +43,11 @@ with open(f'{DATA_FILE}.tmp', 'w', newline='') as csvfile:
             except TypeError:
                 print('ERROR', a)
 
-        e = Entity(name, id=i, bco=bco, logo=img, url=a)
+        e = Entity(name, id=i, bco=bco, logo=str(img), url=str(a))
         writer.writerow(e.to_row())
         i+=1
         bar.next()
     bar.finish()
 
-shutil.move(f'{DATA_FILE}.tmp', DATA_FILE)
+shutil.move(f'{defaults.MAIN_CSV_PATH}.tmp', defaults.MAIN_CSV_PATH)
 print('scrape finished')

python/screenshot.py

@@ -9,7 +9,7 @@ from selenium.webdriver.common.by import By
 
 from common import selectors
 from entity import Entity
-from common import defaults
+from common import defaults,mkdir
 
 options = webdriver.FirefoxOptions()
 options.add_argument("--headless")
@@ -22,18 +22,23 @@ def coord_to_point(c):
 
 driver = webdriver.Firefox(options=options)
 def sc_entity(e: Entity):
-    print(e)
+    print(f'screenshoting: {e}')
+    mkdir.make_dirs([
+            defaults.IMAGES_PATH,
+            defaults.LABELS_PATH,
+    ])
+
     driver.implicitly_wait(10)
     driver.get(e.url)
-    driver.save_screenshot(f"{defaults.DATA_PATH}/{e.bco}.png")
-    driver.save_full_page_screenshot(f"{defaults.DATA_PATH}/{e.bco}.full.png")
+    #driver.save_screenshot(f"{defaults.DATA_PATH}/{e.bco}.png")
+    driver.save_full_page_screenshot(f"{defaults.IMAGES_PATH}/{e.bco}.full.png")
 
     logos = driver.find_elements(By.CSS_SELECTOR, selectors.img_logo) or []
     logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.id_logo) or [])
     logos.extend(driver.find_elements(By.CSS_SELECTOR, selectors.cls_logo) or [])
-    with open(f"{defaults.DATA_PATH}/{e.bco}.full.txt", 'w') as f:
+    with open(f"{defaults.LABELS_PATH}/{e.bco}.full.txt", 'w') as f:
         for i in logos:
-            f.write(f"{e.bco} {coord_to_point(i.rect)}\n")
+            f.write(f"{e.id} {coord_to_point(i.rect)}\n")
 
 if __name__ == '__main__':
     sc_entity(Entity.from_dict({'url': 'http://www.bbva.com.ar', 'bco': 'debug'}))

python/vendor.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import pathlib
 import csv
 import concurrent.futures
 import requests
@@ -7,7 +6,7 @@ import requests
 from progress.bar import ChargingBar
 
 from entity import Entity
-from common import defaults
+from common import defaults,mkdir
 import screenshot
 import web
 
@@ -40,6 +39,4 @@ def from_csv(fn):
 #exit()
 
 if __name__ == '__main__':
-    #pathlib.Path(defaults.DATA_PATH).mkdir(parents=True, exist_ok=True)
-    pathlib.Path(f"{defaults.DATA_PATH}/logos").mkdir(parents=True, exist_ok=True)
-    from_csv(f"{defaults.DATA_PATH}/entidades.csv")
+    from_csv(defaults.MAIN_CSV_PATH)

python/web.py

@@ -5,7 +5,7 @@ import requests
 from bs4 import BeautifulSoup
 
 from entity import Entity
-from common import selectors, defaults
+from common import selectors, defaults, mkdir
 
 def get_page(e: Entity):
     try:
@@ -17,9 +17,10 @@ def get_page(e: Entity):
 
 def get_cert(e: Entity):
     ssl_url = e.url.split("/")[2]
+    mkdir.make_dirs(defaults.CERTS_PATH)
     try:
         cert = ssl.get_server_certificate((ssl_url, 443), ca_certs=None)
-        fn = f"{defaults.DATA_PATH}/{e.bco}.cert"
+        fn = f"{defaults.CERTS_PATH}/{e.bco}.cert"
         with open(fn, 'w') as f:
             f.write(cert)
     except Exception as err:
@@ -39,6 +40,8 @@ def get_logos(e: Entity, page):
     logos.extend(soup.select(selectors.id_logo))
     logos.extend(soup.select(selectors.cls_logo))
 
+    mkdir.make_dirs(defaults.LOGOS_DATA_PATH)
+
     i = 0
     lfn = []
     for l in logos:
@@ -46,7 +49,7 @@ def get_logos(e: Entity, page):
             src = l.attrs['src']
             ext = src.split('.')[-1].split('/')[-1]
             if not src.startswith('http'): src = e.url + src
-            fn = f"{defaults.DATA_PATH}/logos/{e.bco}.{i}.{ext}"
+            fn = f"{defaults.LOGOS_DATA_PATH}/{e.bco}.{i}.{ext}"
             lfn.append(get_img_logo(src, fn))
         i+=1
     return lfn