Commit
·
b55a15f
1
Parent(s):
c2f18d6
Changed timeout
Browse files
scrape.py
CHANGED
|
@@ -111,7 +111,9 @@ async def scrape_page(
|
|
| 111 |
|
| 112 |
try:
|
| 113 |
logger.info(f"Navigating to {url}...")
|
| 114 |
-
await page.goto(url, wait_until="networkidle")
|
|
|
|
|
|
|
| 115 |
response = ScrapeResponse()
|
| 116 |
|
| 117 |
# Always get page title and meta description
|
|
|
|
| 111 |
|
| 112 |
try:
|
| 113 |
logger.info(f"Navigating to {url}...")
|
| 114 |
+
# await page.goto(url, wait_until="networkidle")
|
| 115 |
+
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
| 116 |
+
|
| 117 |
response = ScrapeResponse()
|
| 118 |
|
| 119 |
# Always get page title and meta description
|