Update scraper.py

scraper.py

@@ -2,6 +2,7 @@ from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 import time
 from bs4 import BeautifulSoup
+import tempfile
 
 
 
@@ -10,8 +11,17 @@ def scrape_website(website):
 
     chrome_driver_path = "/usr/bin/chromedriver"
     options = webdriver.ChromeOptions()
-    driver = webdriver.Chrome(service=Service(chrome_driver_path, options=options))
-    
+
+    # Essential arguments for headless environments like Hugging Face
+    options.add_argument("--headless")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+
+    # Use a unique user data directory to prevent session conflicts
+    unique_user_data_dir = tempfile.mkdtemp()
+    options.add_argument(f"--user-data-dir={unique_user_data_dir}")
+
+    driver = webdriver.Chrome(service=Service(chrome_driver_path), options=options)
 
     try:
         driver.get(website)
@@ -19,7 +29,7 @@ def scrape_website(website):
         html = driver.page_source
 
         return html
-    
+
     finally:
         driver.quit()