Update scraper.py

scraper.py

@@ -4,12 +4,11 @@ import time
 import math
 import traceback
 from datetime import datetime
+
 import pandas as pd
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.common.exceptions import (
@@ -49,19 +48,18 @@ def safe_get_text(driver, xpath, retries=1, delay=0.5):
             time.sleep(delay)
     return ""
 
+
 def initialize_driver():
     options = Options()
-    # new headless mode in recent Chrome
+    # use new headless mode
     options.add_argument("--headless=new")
-
     # container flags
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-gpu")
     options.add_argument("--disable-software-rasterizer")
     options.add_argument("--disable-setuid-sandbox")
-
-    # use /tmp for profile & cache
+    # profile & cache in /tmp
     options.add_argument("--remote-debugging-port=9222")
     options.add_argument("--user-data-dir=/tmp/chrome-user-data")
     options.add_argument("--window-size=1920,1080")
@@ -70,12 +68,9 @@ def initialize_driver():
         "AppleWebKit/537.36 (KHTML, like Gecko) "
         "Chrome/135.0.0.0 Safari/537.36"
     )
-
-    # now ChromeDriver is already at /usr/local/bin/chromedriver
-    driver = webdriver.Chrome(options=options)
-    return driver
-
-
+    # Selenium 4 Manager will auto‑download matching driver into SE_CACHE_PATH
+    # (ensure you set ENV SE_CACHE_PATH=/tmp/.cache/selenium in your Dockerfile)
+    return webdriver.Chrome(options=options)
 
 
 def process_batch(start_id, end_id, worker_id):
@@ -106,15 +101,18 @@ def process_batch(start_id, end_id, worker_id):
 
             # wait for title link
             try:
-                wait.until(EC.presence_of_element_located((By.XPATH,
-                    "//a[contains(@class,'DetailPageTitle__thingTitleLink')]")
+                wait.until(EC.presence_of_element_located(
+                    (By.XPATH, "//a[contains(@class,'DetailPageTitle__thingTitleLink')]")
                 ))
             except TimeoutException:
                 continue
 
             title = safe_get_text(driver, "//*[contains(@class,'DetailPageTitle__thingTitleName')]")
             author = safe_get_text(driver, "//a[contains(@class,'DetailPageTitle__thingTitleLink')]")
-            date_posted = safe_get_text(driver, "//a[contains(@class,'DetailPageTitle__thingTitleLink')]/following-sibling::div")
+            date_posted = safe_get_text(
+                driver,
+                "//a[contains(@class,'DetailPageTitle__thingTitleLink')]/following-sibling::div"
+            )
 
             def extract_aria(label):
                 try:
@@ -123,9 +121,10 @@ def process_batch(start_id, end_id, worker_id):
                     return ""
 
             remixes = extract_aria("Remixes")
-            files = extract_aria("Files")
-            makes = extract_aria("Makes")
+            files   = extract_aria("Files")
+            makes   = extract_aria("Makes")
             comments = extract_aria("Comments")
+
             tags = []
             try:
                 tags_el = driver.find_element(By.XPATH, "//*[contains(@class,'TagList__tagList')]")
@@ -159,6 +158,7 @@ def process_batch(start_id, end_id, worker_id):
 
         driver.quit()
         return results
+
     except Exception as e:
         print(f"Worker {worker_id} error: {e}")
         traceback.print_exc()
@@ -168,12 +168,12 @@ def process_batch(start_id, end_id, worker_id):
 def main():
     # configure your range & parallelism
     start_thing = 6993281
-    end_thing = 7003281
+    end_thing   = 7003281
     num_workers = 5
 
     # split work
     total = end_thing - start_thing + 1
-    per = math.ceil(total / num_workers)
+    per   = math.ceil(total / num_workers)
     batches = []
     for i in range(num_workers):
         s = start_thing + i * per
@@ -183,15 +183,16 @@ def main():
     all_results = []
     from concurrent.futures import ThreadPoolExecutor, as_completed
     with ThreadPoolExecutor(max_workers=num_workers) as ex:
-        futures = {ex.submit(process_batch, s, e, wid): (s,e,wid) for s,e,wid in batches}
+        futures = {ex.submit(process_batch, s, e, wid): (s,e,wid)
+                   for s,e,wid in batches}
         for fut in as_completed(futures):
-            res = fut.result()
-            all_results.extend(res)
+            all_results.extend(fut.result())
 
     # upload combined file
     if all_results:
         df_all = pd.DataFrame(all_results)
         upload_df_to_hf(df_all, f"thingiverse_{start_thing}_{end_thing}_all.csv")
 
+
 if __name__ == "__main__":
     main()