Update app.py

app.py

@@ -11,7 +11,6 @@ from datetime import datetime
 from typing import List, Dict, Optional, Union, Tuple
 from pathlib import Path
 from urllib.parse import urlparse, urljoin
-
 import requests
 import validators
 import gradio as gr
@@ -22,6 +21,8 @@ from cleantext import clean
 import qrcode
 from PIL import Image, ImageDraw, ImageFont
 import numpy as np
+import tarfile
+import gzip
 
 # Setup enhanced logging with more detailed formatting
 logging.basicConfig(
@@ -30,8 +31,7 @@ logging.basicConfig(
     handlers=[
         logging.StreamHandler(),
         logging.FileHandler('app.log', encoding='utf-8')
-    ]
-)
+    ])
 logger = logging.getLogger(__name__)
 
 # Ensure output directories exist with modern structure
@@ -43,13 +43,13 @@ for directory in [OUTPUTS_DIR, QR_CODES_DIR, TEMP_DIR]:
 
 class EnhancedURLProcessor:
     """Advanced URL processing with complete content extraction"""
-    
+
     def __init__(self):
         self.session = requests.Session()
         self.timeout = 15  # Extended timeout for larger content
         self.max_retries = 3
         self.user_agent = UserAgent()
-        
+
         # Enhanced headers for better site compatibility
         self.session.headers.update({
             'User-Agent': self.user_agent.random,
@@ -70,11 +70,9 @@ class EnhancedURLProcessor:
         try:
             if not validators.url(url):
                 return {'is_valid': False, 'message': 'Invalid URL format', 'details': 'URL must begin with http:// or https://'}
-
             parsed = urlparse(url)
             if not all([parsed.scheme, parsed.netloc]):
                 return {'is_valid': False, 'message': 'Incomplete URL', 'details': 'Missing scheme or domain'}
-
             # Try HEAD request first to check accessibility
             try:
                 head_response = self.session.head(url, timeout=5)
@@ -100,19 +98,18 @@ class EnhancedURLProcessor:
         """Enhanced content fetcher with retry mechanism and complete character extraction"""
         try:
             logger.info(f"Fetching content from URL: {url} (Attempt {retry_count + 1}/{self.max_retries})")
-            
+
             # Update User-Agent randomly for each request
             self.session.headers.update({'User-Agent': self.user_agent.random})
-            
+
             response = self.session.get(url, timeout=self.timeout)
             response.raise_for_status()
-            
+
             # Detect encoding
             if response.encoding is None:
                 encoding = chardet.detect(response.content)['encoding'] or 'utf-8'
             else:
                 encoding = response.encoding
-
             # Decode content with fallback
             try:
                 raw_content = response.content.decode(encoding, errors='replace')
@@ -136,13 +133,11 @@ class EnhancedURLProcessor:
                 processed_content = self._process_html_content(raw_content, url)
             else:
                 processed_content = raw_content
-
             return {
                 'content': processed_content,
                 'raw_content': raw_content,
                 'metadata': metadata
             }
-
         except requests.exceptions.RequestException as e:
             if retry_count < self.max_retries - 1:
                 logger.warning(f"Retry {retry_count + 1}/{self.max_retries} for URL: {url}")
@@ -158,7 +153,7 @@ class EnhancedURLProcessor:
         """Process HTML content while preserving all characters"""
         try:
             soup = BeautifulSoup(content, 'html.parser')
-            
+
             # Convert relative URLs to absolute
             for tag in soup.find_all(['a', 'img', 'link', 'script']):
                 for attr in ['href', 'src']:
@@ -167,12 +162,10 @@ class EnhancedURLProcessor:
                             tag[attr] = urljoin(base_url, tag[attr])
                         except Exception:
                             pass
-
             # Extract all text content
             text_parts = []
             for element in soup.stripped_strings:
                 text_parts.append(str(element))
-
             return '\n'.join(text_parts)
         except Exception as e:
             logger.error(f"HTML processing error: {e}")
@@ -184,7 +177,7 @@ class EnhancedFileProcessor:
     def __init__(self, max_file_size: int = 5 * 1024 * 1024 * 1024):  # 5GB default
         self.max_file_size = max_file_size
         self.supported_extensions = {
-            '.txt', '.md', '.csv', '.json', '.xml', '.html', '.htm', 
+            '.txt', '.md', '.csv', '.json', '.xml', '.html', '.htm',
             '.log', '.yml', '.yaml', '.ini', '.conf', '.cfg',
             '.zip', '.tar', '.gz', '.bz2', '.7z', '.rar',
             '.pdf', '.doc', '.docx', '.rtf', '.odt'
@@ -204,17 +197,18 @@ class EnhancedFileProcessor:
 
             with tempfile.TemporaryDirectory() as temp_dir:
                 temp_dir_path = Path(temp_dir)
-                
+
                 # Handle different archive types
                 if self._is_archive(file.name):
                     dataset.extend(self._process_archive(file.name, temp_dir_path))
-                else:
+                elif Path(file.name).suffix.lower() in self.supported_extensions:
                     dataset.extend(self._process_single_file(file))
+                else:
+                    logger.warning(f"Unsupported file type: {file.name}")
 
         except Exception as e:
             logger.error(f"Error processing file: {str(e)}")
             return []
-
         return dataset
 
     def _is_archive(self, filepath: str) -> bool:
@@ -224,14 +218,14 @@ class EnhancedFileProcessor:
         ])
 
     def _process_single_file(self, file) -> List[Dict]:
-        """Process a single file with enhanced character extraction"""
+        """Process a single file with enhanced character extraction and JSON handling"""
         try:
             file_stat = os.stat(file.name)
             file_size = file_stat.st_size
-            
+
             # Initialize content storage
             content_parts = []
-            
+
             # Process file in chunks for large files
             chunk_size = 10 * 1024 * 1024  # 10MB chunks
             with open(file.name, 'rb') as f:
@@ -239,7 +233,7 @@ class EnhancedFileProcessor:
                     chunk = f.read(chunk_size)
                     if not chunk:
                         break
-                        
+
                     # Detect encoding for each chunk
                     encoding = chardet.detect(chunk)['encoding'] or 'utf-8'
                     try:
@@ -252,6 +246,43 @@ class EnhancedFileProcessor:
             # Combine all chunks
             complete_content = ''.join(content_parts)
 
+            # Check if the content is valid JSON regardless of file extension
+            try:
+                if mimetypes.guess_type(file.name)[0] == 'application/json' or file.name.lower().endswith('.json'):
+                    # It's a JSON file by type or extension
+                    json_data = json.loads(complete_content)
+                    return [{
+                        'source': 'json_file',
+                        'filename': os.path.basename(file.name),
+                        'file_size': file_size,
+                        'mime_type': 'application/json',
+                        'created': datetime.fromtimestamp(file_stat.st_ctime).isoformat(),
+                        'modified': datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
+                        'content': json_data,  # Store the parsed JSON object
+                        'raw_content': complete_content,  # Store the original JSON string
+                        'timestamp': datetime.now().isoformat()
+                    }]
+                else:
+                    # Try to parse as JSON anyway
+                    try:
+                        json_data = json.loads(complete_content)
+                        # If we get here, it's valid JSON despite the extension
+                        return [{
+                            'source': 'json_content',
+                            'filename': os.path.basename(file.name),
+                            'file_size': file_size,
+                            'mime_type': 'application/json',
+                            'created': datetime.fromtimestamp(file_stat.st_ctime).isoformat(),
+                            'modified': datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
+                            'content': json_data,  # Store the parsed JSON object
+                            'raw_content': complete_content,  # Store the original JSON string
+                            'timestamp': datetime.now().isoformat()
+                        }]
+                    except json.JSONDecodeError:
+                        logger.warning(f"File {file.name} is not valid JSON.")
+            except Exception as e:
+                logger.error(f"Error during JSON processing: {e}")
+
             return [{
                 'source': 'file',
                 'filename': os.path.basename(file.name),
@@ -280,22 +311,46 @@ class EnhancedFileProcessor:
                             if extracted_path.suffix.lower() in self.supported_extensions:
                                 with open(extracted_path, 'rb') as f:
                                     dataset.extend(self._process_single_file(f))
+            # Handle TAR archives
+            elif archive_path.lower().endswith(('.tar', '.tar.gz', '.tgz')):
+                try:
+                    with tarfile.open(archive_path, 'r:*') as tar_ref:
+                        for member in tar_ref.getmembers():
+                            if member.isfile():
+                                extracted_path = extract_to / member.name
+                                tar_ref.extract(member, path=extract_to)
+                                if extracted_path.suffix.lower() in self.supported_extensions:
+                                    with open(extracted_path, 'rb') as f:
+                                        dataset.extend(self._process_single_file(f))
+                except tarfile.TarError as e:
+                    logger.error(f"Error processing TAR archive: {e}")
+            # Handle GZIP archives (single file)
+            elif archive_path.lower().endswith('.gz'):
+                extracted_path = extract_to / Path(archive_path).stem
+                try:
+                    with gzip.open(archive_path, 'rb') as gz_file, open(extracted_path, 'wb') as outfile:
+                        outfile.write(gz_file.read())
+                    if extracted_path.suffix.lower() in self.supported_extensions:
+                        with open(extracted_path, 'rb') as f:
+                            dataset.extend(self._process_single_file(f))
+                except gzip.GzipFile as e:
+                    logger.error(f"Error processing GZIP archive: {e}")
+            # TODO: Add support for other archive types (.bz2, .7z, .rar) - may require external libraries
+            elif archive_path.lower().endswith(('.bz2', '.7z', '.rar')):
+                logger.warning(f"Support for {Path(archive_path).suffix} archives is not yet fully implemented.")
 
-            # TODO: Add support for other archive types (tar, 7z, etc.)
-            
         except Exception as e:
             logger.error(f"Archive processing error: {e}")
-        
         return dataset
 
     def chunk_data(self, data: Union[Dict, List], max_size: int = 2953) -> List[Dict]:
+        """Enhanced data chunking with sequence metadata"""
         try:
-            # Convert data to JSON bytes
+            # Convert data to JSON string
             json_str = json.dumps(data, ensure_ascii=False)
-            json_bytes = json_str.encode('utf-8')
-            total_length = len(json_bytes)
-    
-            # Calculate metadata overhead in bytes
+            total_length = len(json_str)
+
+            # Calculate overhead for metadata
             metadata_template = {
                 "chunk_index": 0,
                 "total_chunks": 1,
@@ -303,48 +358,32 @@ class EnhancedFileProcessor:
                 "chunk_hash": "",
                 "data": ""
             }
-            overhead_bytes = len(json.dumps(metadata_template).encode('utf-8')) + 20  # Add padding
-    
-            effective_chunk_size = max_size - overhead_bytes
-    
-            if effective_chunk_size <= 0:
-                raise ValueError("Max size is too small after accounting for metadata overhead")
-    
-            chunks = []
-            start = 0
-            while start < total_length:
-                end = start + effective_chunk_size
-                # Ensure valid Unicode by decoding
-                chunk_str = json_bytes[start:end].decode('utf-8', errors='replace')
+            overhead = len(json.dumps(metadata_template)) + 20  # Extra padding for safety
+
+            # Calculate effective chunk size
+            effective_chunk_size = max_size - overhead
+
+            if total_length <= effective_chunk_size:
+                # Data fits in one chunk
                 chunk = {
-                    "chunk_index": len(chunks),
-                    "total_chunks": -1,  # To be set later
+                    "chunk_index": 0,
+                    "total_chunks": 1,
                     "total_length": total_length,
-                    "chunk_hash": hash(chunk_str) & 0xFFFFFFFF,
-                    "data": chunk_str
+                    "chunk_hash": hash(json_str) & 0xFFFFFFFF,  # 32-bit hash
+                    "data": json_str
                 }
-                chunks.append(chunk)
-                start = end
-    
-            # Update total_chunks in each chunk
-            for i, chunk in enumerate(chunks):
-                chunk["total_chunks"] = len(chunks)
-    
-            return chunks
-        except Exception as e:
-            logger.error(f"Error chunking data: {e}")
-            return []
-            
+                return [chunk]
+
             # Calculate number of chunks needed
             num_chunks = -(-total_length // effective_chunk_size)  # Ceiling division
             chunk_size = -(-total_length // num_chunks)  # Even distribution
-            
+
             chunks = []
             for i in range(num_chunks):
                 start_idx = i * chunk_size
                 end_idx = min(start_idx + chunk_size, total_length)
                 chunk_data = json_str[start_idx:end_idx]
-                
+
                 chunk = {
                     "chunk_index": i,
                     "total_chunks": num_chunks,
@@ -353,58 +392,56 @@ class EnhancedFileProcessor:
                     "data": chunk_data
                 }
                 chunks.append(chunk)
-            
+
             return chunks
-            
         except Exception as e:
             logger.error(f"Error chunking data: {e}")
             return []
 
-def generate_stylish_qr(data: Union[str, Dict], 
-                       filename: str,
-                       size: int = 10,
-                       border: int = 4,
-                       fill_color: str = "#000000",
-                       back_color: str = "#FFFFFF") -> str:
+def generate_stylish_qr(data: Union[str, Dict],
+                        filename: str,
+                        size: int = 10,
+                        border: int = 4,
+                        fill_color: str = "#000000",
+                        back_color: str = "#FFFFFF") -> str:
     """Generate a stylish QR code with enhanced visual appeal"""
     try:
         qr = qrcode.QRCode(
             version=None,
-            error_correction=qrcode.constants.ERROR_CORRECT_M,
+            error_correction=qrcode.constants.ERROR_CORRECT_H,
             box_size=size,
             border=border
         )
-        
+
         # Add data to QR code
         if isinstance(data, dict):
             qr.add_data(json.dumps(data, ensure_ascii=False))
         else:
             qr.add_data(data)
-        
+
         qr.make(fit=True)
-        
+
         # Create QR code image with custom colors
         qr_image = qr.make_image(fill_color=fill_color, back_color=back_color)
-        
+
         # Convert to RGBA for transparency support
         qr_image = qr_image.convert('RGBA')
-        
+
         # Add subtle gradient overlay
         gradient = Image.new('RGBA', qr_image.size, (0, 0, 0, 0))
         draw = ImageDraw.Draw(gradient)
         for i in range(qr_image.width):
             alpha = int(255 * (1 - i/qr_image.width) * 0.1)  # 10% maximum opacity
             draw.line([(i, 0), (i, qr_image.height)], fill=(255, 255, 255, alpha))
-        
+
         # Combine images
         final_image = Image.alpha_composite(qr_image, gradient)
-        
+
         # Save the image
         output_path = QR_CODES_DIR / filename
         final_image.save(output_path, quality=95)
-        
+
         return str(output_path)
-        
     except Exception as e:
         logger.error(f"QR generation error: {e}")
         return ""
@@ -414,7 +451,7 @@ def generate_qr_codes(data: Union[str, Dict, List], combined: bool = True) -> Li
     try:
         file_processor = EnhancedFileProcessor()
         paths = []
-        
+
         if combined:
             # Process combined data
             chunks = file_processor.chunk_data(data)
@@ -455,8 +492,7 @@ def generate_qr_codes(data: Union[str, Dict, List], combined: bool = True) -> Li
                     )
                     if qr_path:
                         paths.append(qr_path)
-        
-        return paths
+                return paths
     except Exception as e:
         logger.error(f"QR code generation error: {e}")
         return []
@@ -553,9 +589,7 @@ def create_modern_interface():
         with gr.Tab("📁 File Input"):
             file_input = gr.File(
                 label="Upload Files",
-                file_types=["text", ".zip", ".json",
-                            ".tar", ".gz", ".bz2", ".7z", ".rar",
-                            ".pdf", ".doc", ".docx", ".rtf", ".odt"],
+                file_types=["*"],  # Accept all file types
                 file_count="multiple"
             )
         with gr.Tab("📋 JSON Input"):
@@ -695,15 +729,15 @@ def create_modern_interface():
         gr.Markdown("""
         ### 🚀 Features
                 - **Complete URL Scraping**: Extracts every character from web pages
-                - **Advanced File Processing**: Full content extraction from various text-based files and common archives
-                - **Smart JSON Handling**: Processes any size JSON with automatic chunking
+                - **Advanced File Processing**: Full content extraction from various text-based files and common archives. Supports flexible JSON handling.
+                - **Smart JSON Handling**: Processes any size JSON with automatic chunking, either via direct input or file upload.
                 - **Sequential QR Codes**: Maintains data integrity across multiple codes
                 - **Modern Design**: Clean, responsive interface with visual feedback
 
                 ### 💡 Tips
                 1. **URLs**: Enter multiple URLs separated by commas or newlines
-                2. **Files**: Upload various text-based files (e.g., .txt, .md, .csv), JSON files (.json), or common archives (.zip, .tar, .gz)
-                3. **JSON**: Use the example button to see the expected format or upload a .json file
+                2. **Files**: Upload any type of file. The processor will attempt to handle supported text-based files, archives (.zip, .tar, .gz), and JSON files.
+                3. **JSON**: Use the example button to see the expected format or upload a .json file. The system will also try to detect JSON content in other file types.
                 4. **QR Codes**: Choose whether to combine data into sequential codes
                 5. **Processing**: Monitor the status for real-time feedback