update the code to handle ads and cloudflare challenge fixed2

app.py

@@ -260,29 +260,36 @@ def create_browser_context(playwright):
     )
 
 def setup_request_interception(page):
-    redirect_count = 0
+    redirect_urls = set()
     
     def handle_request(route):
-        nonlocal redirect_count
         request = route.request
+        url = request.url
         
         # Block known ad/tracking patterns
-        if any(pattern in request.url.lower() for pattern in BLOCK_PATTERNS):
-            print(f"Blocking request to: {request.url}")
+        if any(pattern in url.lower() for pattern in BLOCK_PATTERNS):
+            print(f"Blocking request to: {url}")
             route.abort()
             return
-
-        # Handle redirects
-        if request.redirect_chain:
-            redirect_count += 1
-            if redirect_count > MAX_REDIRECTS:
-                print(f"Too many redirects ({redirect_count}), aborting request")
-                route.abort()
-                return
-        
+            
+        # Track potential redirects by monitoring navigation requests
+        if request.resource_type == "document":
+            if url in redirect_urls:
+                if len(redirect_urls) > MAX_REDIRECTS:
+                    print(f"Too many redirects (>{MAX_REDIRECTS}), aborting request")
+                    route.abort()
+                    return
+            redirect_urls.add(url)
+            
         # Continue with the request
         route.continue_()
     
+    # Listen for response events to detect redirects
+    def handle_response(response):
+        if response.status >= 300 and response.status <= 399:
+            redirect_urls.add(response.url)
+    
+    page.on("response", handle_response)
     page.route("**/*", handle_request)
 
 def take_screenshot(url):
@@ -305,17 +312,22 @@ def take_screenshot(url):
                 print("Attempting to navigate to URL...")
                 response = page.goto(
                     url,
-                    wait_until="commit",  # Changed to commit instead of domcontentloaded
+                    wait_until="commit",
                     timeout=PAGE_TIMEOUT
                 )
                 
                 if not response:
                     print("No response received, attempting to continue...")
+                elif response.status >= 300 and response.status <= 399:
+                    print(f"Received redirect status code: {response.status}")
                 elif response.status >= 400:
                     print(f"Received error status code: {response.status}")
                 
                 # Try to wait for the page to be more stable
-                wait_for_page_stable(page)
+                try:
+                    wait_for_page_stable(page)
+                except Exception as e:
+                    print(f"Page stability warning: {e}")
                 
                 # Take screenshot even if page might not be fully loaded
                 print("Taking screenshot...")
@@ -333,7 +345,10 @@ def take_screenshot(url):
                 except:
                     raise nav_error
             finally:
-                context.close()
+                try:
+                    context.close()
+                except Exception as close_error:
+                    print(f"Warning: Error while closing context: {close_error}")
         
         if os.path.exists(filepath):
             print(f"Screenshot saved successfully to {filepath}")