Update app.py

app.py

@@ -167,68 +167,56 @@ class FileProcessor:
         """Check if the file is a text file based on its extension."""
         return any(file_path.lower().endswith(ext) for ext in self.supported_text_extensions)
 
-    def process_files(self, files: Union[List[gr.File], List[str]]) -> List[Dict]:
-        """Process multiple uploaded files and return a single JSON extraction"""
-        if not files:
-            return []
-
+    
+    def validate_filepath(path: Path) -> bool:
+        """Validate file exists and has supported extension"""
+        try:
+            return path.exists() and path.is_file() and path.suffix.lower() in valid_extensions
+        except Exception as e:
+            logger.error(f"Validation error for {path}: {str(e)}")
+            return False
+    
+    def process_files(base_path: str = "/app/data") -> list:
+        """Process files with validation and error handling"""
         combined_data = []
-        self.processed_zip_count = 0
         
-        try:
-            for file in files:
-                file_path = file.name if isinstance(file, gr.File) else file
-                
-                logger.info(f"Processing file: {file_path}")
-                
-                if os.path.isdir(file_path):
-                    logger.warning(f"Skipping directory: {file_path}")
+        base_dir = Path(base_path)
+        if not base_dir.exists():
+            base_dir.mkdir(parents=True, exist_ok=True)
+            logger.info(f"Created data directory at {base_dir}")
+    
+        for item in base_dir.glob('**/*'):
+            try:
+                # Skip directories immediately
+                if item.is_dir():
+                    logger.debug(f"Skipping directory: {item}")
                     continue
-                
-                if not os.path.exists(file_path):
-                    logger.warning(f"File does not exist: {file_path}")
+    
+                # Validate file using shared function
+                if not validate_filepath(item):
+                    logger.warning(f"Invalid file skipped: {item}")
                     continue
+    
+                logger.info(f"Processing valid file: {item.name}")
                 
-                file_size = os.path.getsize(file_path)
-                if file_size > self.max_file_size:
-                    logger.warning(f"File size ({file_size} bytes) exceeds maximum allowed size")
-                    continue
-
-                if zipfile.is_zipfile(file_path):
-                    if self.processed_zip_count >= self.max_zip_files:
-                        logger.warning(f"Maximum number of ZIP files ({self.max_zip_files}) reached, skipping {file_path}")
-                        continue
-                    self.processed_zip_count += 1
-                    zip_results = self._process_zip_file(file_path)
-                    combined_data.extend(zip_results)
-                elif self.is_text_file(file_path):
-                    file_results = self._process_single_file(file) # Changed file_path to file
-                    combined_data.extend(file_results)
-                else:
-                    logger.warning(f"Unsupported file type: {file_path}")
-
-        except Exception as e:
-            logger.error(f"Error processing files: {str(e)}")
-        
+                # Add actual processing logic here
+                file_data = process_single_file(item)  # Your processing function
+                combined_data.append(file_data)
+    
+            except Exception as e:
+                logger.error(f"Failed processing {item}: {str(e)}")
+                continue
+    
         return combined_data
-
     
-    def _process_single_file(self, file) -> List[Dict]:
-        try:
-            file_stat = os.stat(file.name)
-            with open(file.name, 'r', encoding='utf-8', errors='ignore') as f:
-                content = f.read()
-            
-            return [{
-                'source': 'file',
-                'filename': os.path.basename(file.name),
-                'file_size': file_stat.st_size,
-                'mime_type': mimetypes.guess_type(file.name)[0],
-                'created': datetime.fromtimestamp(file_stat.st_ctime).isoformat(),
-                'modified': datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
-                'content': content,
-                'timestamp': datetime.now().isoformat()
-            }]
+    def process_single_file(file_path: Path) -> dict:
+        """Example processing function"""
+        # Add your actual file processing logic here
+        return {
+            'filename': file_path.name,
+            'content': "processed content",  # Replace with real content
+            'metadata': {}  # Add actual metadata
+        }
         except Exception as e:
             logger.error(f"File processing error: {e}")
             return []