Update scraper.py

scraper.py

@@ -49,35 +49,35 @@ def safe_get_text(driver, xpath, retries=1, delay=0.5):
             time.sleep(delay)
     return ""
 
-
-
 def initialize_driver():
     options = Options()
-    options.headless = True
+    # new headless mode in recent Chrome
+    options.add_argument("--headless=new")
 
-    # Linux container flags
+    # container flags
     options.add_argument("--disable-dev-shm-usage")
     options.add_argument("--no-sandbox")
     options.add_argument("--disable-gpu")
     options.add_argument("--disable-software-rasterizer")
     options.add_argument("--disable-setuid-sandbox")
 
-    # Use /tmp for profile & remote-debugging
+    # use /tmp for profile & cache
     options.add_argument("--remote-debugging-port=9222")
     options.add_argument("--user-data-dir=/tmp/chrome-user-data")
-    options.add_argument("window-size=1920,1080")
+    options.add_argument("--window-size=1920,1080")
     options.add_argument(
-        "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-        "(KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+        "user-agent=Mozilla/5.0 (X11; Linux x86_64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
     )
 
-    # webdriver-manager will download the correct ChromeDriver version
-    service = Service(ChromeDriverManager(path="/tmp/.wdm").install())
-    driver = webdriver.Chrome(service=service, options=options)
+    # now ChromeDriver is already at /usr/local/bin/chromedriver
+    driver = webdriver.Chrome(options=options)
     return driver
 
 
 
+
 def process_batch(start_id, end_id, worker_id):
     print(f"Worker {worker_id} processing IDs {start_id} to {end_id}")
     try: