Update app.py

app.py

@@ -39,37 +39,13 @@ class URLProcessor:
         self.session = requests.Session()
         self.timeout = 10  # seconds
         self.session.headers.update({
-            'User -Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
             'Accept-Language': 'en-US,en;q=0.5',
             'Accept-Encoding': 'gzip, deflate, br',
             'Connection': 'keep-alive',
             'Upgrade-Insecure-Requests': '1'
         })
-    def advanced_text_cleaning(self, text: str) -> str:
-        """Robust text cleaning with version compatibility"""
-        try:
-            cleaned_text = clean(
-                text,
-                fix_unicode=True,
-                to_ascii=True,
-                lower=True,
-                no_line_breaks=True,
-                no_urls=False,
-                no_emails=True,
-                no_phone_numbers=True,
-                no_numbers=False,
-                no_digits=False,
-                no_currency_symbols=True,
-                no_punct=False
-            ).strip()
-            return cleaned_text
-        except Exception as e:
-            logger.warning(f"Text cleaning error: {e}. Using fallback method.")
-            text = re.sub(r'[\x00-\x1F\x7F-\x9F]', '', text)  # Remove control characters
-            text = text.encode('ascii', 'ignore').decode('ascii')  # Remove non-ASCII characters
-            text = re.sub(r'\s+', ' ', text)  # Normalize whitespace
-            return text.strip()
 
     def validate_url(self, url: str) -> Dict:
         """Validate URL format and accessibility"""
@@ -77,51 +53,77 @@ class URLProcessor:
             if not validators.url(url):
                 return {'is_valid': False, 'message': 'Invalid URL format'}
 
-            response = self.session.head(url, timeout=self.timeout)
+            # Try a simple GET request to check if the URL is accessible
+            response = self.session.get(url, timeout=self.timeout)
             response.raise_for_status()
             return {'is_valid': True, 'message': 'URL is valid and accessible'}
         except Exception as e:
             return {'is_valid': False, 'message': f'URL validation failed: {str(e)}'}
 
     def fetch_content(self, url: str) -> Optional[Dict]:
-        """Universal content fetcher with special case handling"""
+        """Simple content fetcher that returns the raw HTML"""
         try:
-            logger.info(f"Fetching content from URL: {url}")  # Log the URL being fetched
+            logger.info(f"Fetching content from URL: {url}")
             response = self.session.get(url, timeout=self.timeout)
-            response.raise_for_status()  # Raise an error for bad responses
-
-            soup = BeautifulSoup(response.text, 'html.parser')
-
-            # Remove unwanted elements
-            for element in soup(['script', 'style', 'nav', 'footer', 'header', 'meta', 'link']):
-                element.decompose()
-
-            # Extract main content
-            main_content = soup.find('main') or soup.find('article') or soup.body
-
-            if main_content is None:
-                logger.warning(f"No main content found for URL: {url}")
-                return {
-                    'content': response.text,  # Return the full HTML if no main content found
-                    'content_type': response.headers.get('Content-Type', ''),
-                    'timestamp': datetime.now().isoformat()
-                }
-
-            # Clean and structure content
-            text_content = main_content.get_text(separator='\n', strip=True)
-            cleaned_content = self.advanced_text_cleaning(text_content)
-
+            response.raise_for_status()
+            
+            # Return the raw HTML content
             return {
-                'content': cleaned_content,
+                'content': response.text,
                 'content_type': response.headers.get('Content-Type', ''),
                 'timestamp': datetime.now().isoformat()
             }
-        except requests.RequestException as e:
-            logger.error(f"Request failed: {e}")
-            return None
         except Exception as e:
             logger.error(f"Content fetch failed: {e}")
             return None
+    
+    def process_all_inputs(urls, file, text, combine):
+        """Process all input types and generate QR codes"""
+        try:
+            results = []
+            file_processor = FileProcessor()  # Initialize file_processor here
+    
+            # Process text input first (since it's direct JSON)
+            if text and text.strip():
+                try:
+                    json_data = json.loads(text)
+                    if isinstance(json_data, list):
+                        results.extend(json_data)
+                    else:
+                        results.append(json_data)
+                except json.JSONDecodeError as e:
+                    return None, [], f"❌ Invalid JSON format: {str(e)}"
+    
+            # Process URLs if provided
+            if urls and urls.strip():
+                processor = URLProcessor()
+                url_list = re.split(r'[,\n]', urls)
+                url_list = [url.strip() for url in url_list if url.strip()]
+    
+                for url in url_list:
+                    logger.info(f"Processing URL: {url}")
+                    validation = processor.validate_url(url)
+                    
+                    if validation.get('is_valid'):
+                        logger.info(f"URL {url} is valid, fetching content...")
+                        content = processor.fetch_content(url)
+                        
+                        if content:
+                            logger.info(f"Content fetched successfully from {url}") results.append(content['content'])
+                        else:
+                            logger.warning(f"Failed to fetch content from {url}")
+                    else:
+                        logger.error(f"Invalid URL: {validation.get('message')}")
+    
+            # Combine results if needed
+            if combine:
+                combined_content = "\n".join(results)
+                return combined_content, results, None
+    
+            return results, [], None
+        except Exception as e:
+            logger.error(f"Error processing inputs: {e}")
+            return None, [], f"❌ An error occurred: {str(e)}"
             
     def _handle_google_drive(self, url: str) -> Optional[Dict]:
         """Process Google Drive file links"""