Update app.py

app.py

@@ -139,8 +139,14 @@ class URLProcessor:
             # Try with a different user agent if it's a social media site
             if any(domain in url for domain in ['facebook.com', 'instagram.com', 'twitter.com', 'x.com', 'huggingface.co']):
                 self.session.headers.update({
-                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
+                    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36',
+                    # Add cookie consent headers to bypass some login walls
+                    'Cookie': 'c_user=0; xs=0; datr=0; locale=en_US; wd=1920x1080'
                 })
+                # For Facebook, try to access the mobile version which often has fewer restrictions
+                if 'facebook.com' in url and 'm.facebook.com' not in url:
+                    url = url.replace('www.facebook.com', 'm.facebook.com')
+                    logger.info(f"Switched to mobile Facebook URL: {url}")
             
             response = self.session.get(url, timeout=self.timeout)
             response.raise_for_status()
@@ -148,8 +154,10 @@ class URLProcessor:
             logger.info(f"Response status: {response.status_code}, Content-Type: {response.headers.get('Content-Type')}")
             
             # Save the raw HTML for debugging if needed
-            with open(f"debug_raw_{int(time.time())}.html", "w", encoding="utf-8") as f:
+            debug_path = f"/Users/a2014/urld/debug_raw_{int(time.time())}.html"
+            with open(debug_path, "w", encoding="utf-8") as f:
                 f.write(response.text)
+            logger.info(f"Saved raw HTML to {debug_path}")
             
             soup = BeautifulSoup(response.text, 'html.parser')
 
@@ -157,17 +165,72 @@ class URLProcessor:
             for element in soup(['script', 'style', 'nav', 'footer', 'header', 'meta', 'link']):
                 element.decompose()
                 
-            # Remove login walls and overlays common on social media sites
-            for element in soup.select('.login-wall, .signup-wall, .overlay, .modal, [role="dialog"], [aria-modal="true"]'):
-                element.decompose()
+            # Simulate "ESC key" by removing login walls and overlays common on social media sites
+            login_wall_selectors = [
+                '.login-wall', '.signup-wall', '.overlay', '.modal', 
+                '[role="dialog"]', '[aria-modal="true"]', '.login-overlay', 
+                '.signup-overlay', '.uiLayer', '.fb_overlay', '.ReactModalPortal',
+                '[data-testid="login_dialog"]', '[data-testid="signup_dialog"]',
+                '.login-signup-modal', '.onboarding-modal', '.signup-wrapper',
+                '.login-wrapper', '.login-container', '.signup-container',
+                '.login-modal', '.signup-modal', '.auth-modal', '.auth-wall'
+            ]
+            for selector in login_wall_selectors:
+                for element in soup.select(selector):
+                    logger.info(f"Removing login wall element: {selector}")
+                    element.decompose()
                 
-            # Remove specific elements for known sites
+            # Enhanced removal for social media sites
             if 'facebook.com' in url:
-                for element in soup.select('[data-testid="cookie-policy-manage-dialog"], [role="banner"], [role="complementary"]'):
-                    element.decompose()
-            elif 'instagram.com' in url or 'twitter.com' in url or 'x.com' in url:
-                for element in soup.select('[role="presentation"], [role="banner"], [role="complementary"]'):
-                    element.decompose()
+                # Facebook specific elements - simulating ESC key
+                fb_selectors = [
+                    '[data-testid="cookie-policy-manage-dialog"]', 
+                    '[role="banner"]', '[role="complementary"]', 
+                    '.login_form_container', '.login_form', '#login_form', 
+                    '.uiLayer', '.pluginConnectButton', '.fbPageBanner', 
+                    '._5hn6', '._67m7', '.nonLoggedInSignUp', 
+                    '#headerArea', '.uiContextualLayer', '.uiContextualLayerPositioner'
+                ]
+                for selector in fb_selectors:
+                    for element in soup.select(selector):
+                        element.decompose()
+                
+                # Look for the main content in mobile version
+                main_content = soup.select_one('#m_story_permalink_view') or soup.select_one('#mobile_injected_video_feed_pagelet')
+                if main_content:
+                    logger.info("Found Facebook mobile main content")
+                    
+            elif 'instagram.com' in url:
+                # Instagram specific elements - simulating ESC key
+                ig_selectors = [
+                    '[role="presentation"]', '[role="banner"]', '[role="complementary"]', 
+                    '.RnEpo', '._acb3', '._ab8w', '._abn5', '.x1n2onr6', 
+                    '.x78zum5', '.x1q0g3np', '.xieb3on', '._a9-z', '._a9_1', 
+                    '._aa4b', '.x1i10hfl', '.x9f619', '.xnz67gz', '.x78zum5',
+                    '.x1q0g3np', '.x1gslohp', '.xieb3on', '.x1lcm9me'
+                ]
+                for selector in ig_selectors:
+                    for element in soup.select(selector):
+                        element.decompose()
+                
+                # Try to find the main content
+                insta_content = soup.select_one('main article') or soup.select_one('._aagv') or soup.select_one('._ab1y')
+                if insta_content:
+                    logger.info("Found Instagram main content")
+                    
+            elif 'twitter.com' in url or 'x.com' in url:
+                # X/Twitter already works well for public content, but clean up any remaining overlays
+                x_selectors = [
+                    '[data-testid="LoginForm"]', '[data-testid="SignupForm"]',
+                    '[data-testid="sheetDialog"]', '[data-testid="mask"]',
+                    '.r-zchlnj', '.r-1xcajam', '.r-1d2f490', '.r-1p0dtai', 
+                    '.r-1pi2tsx', '.r-u8s1d', '.css-175oi2r', '.css-1dbjc4n', 
+                    '.r-kemksi', '[data-testid="BottomBar"]'
+                ]
+                for selector in x_selectors:
+                    for element in soup.select(selector):
+                        element.decompose()
+                        
             elif 'huggingface.co' in url:
                 # Special handling for Hugging Face
                 logger.info("Applying special handling for Hugging Face")