Update app.py

app.py

@@ -289,13 +289,13 @@ class EnhancedFileProcessor:
         return dataset
 
     def chunk_data(self, data: Union[Dict, List], max_size: int = 2953) -> List[Dict]:
-        """Enhanced data chunking with sequence metadata"""
         try:
-            # Convert data to JSON string
+            # Convert data to JSON bytes
             json_str = json.dumps(data, ensure_ascii=False)
-            total_length = len(json_str)
-            
-            # Calculate overhead for metadata
+            json_bytes = json_str.encode('utf-8')
+            total_length = len(json_bytes)
+    
+            # Calculate metadata overhead in bytes
             metadata_template = {
                 "chunk_index": 0,
                 "total_chunks": 1,
@@ -303,21 +303,37 @@ class EnhancedFileProcessor:
                 "chunk_hash": "",
                 "data": ""
             }
-            overhead = len(json.dumps(metadata_template)) + 20  # Extra padding for safety
-            
-            # Calculate effective chunk size
-            effective_chunk_size = max_size - overhead
-            
-            if total_length <= effective_chunk_size:
-                # Data fits in one chunk
+            overhead_bytes = len(json.dumps(metadata_template).encode('utf-8')) + 20  # Add padding
+    
+            effective_chunk_size = max_size - overhead_bytes
+    
+            if effective_chunk_size <= 0:
+                raise ValueError("Max size is too small after accounting for metadata overhead")
+    
+            chunks = []
+            start = 0
+            while start < total_length:
+                end = start + effective_chunk_size
+                # Ensure valid Unicode by decoding
+                chunk_str = json_bytes[start:end].decode('utf-8', errors='replace')
                 chunk = {
-                    "chunk_index": 0,
-                    "total_chunks": 1,
+                    "chunk_index": len(chunks),
+                    "total_chunks": -1,  # To be set later
                     "total_length": total_length,
-                    "chunk_hash": hash(json_str) & 0xFFFFFFFF,  # 32-bit hash
-                    "data": json_str
+                    "chunk_hash": hash(chunk_str) & 0xFFFFFFFF,
+                    "data": chunk_str
                 }
-                return [chunk]
+                chunks.append(chunk)
+                start = end
+    
+            # Update total_chunks in each chunk
+            for i, chunk in enumerate(chunks):
+                chunk["total_chunks"] = len(chunks)
+    
+            return chunks
+        except Exception as e:
+            logger.error(f"Error chunking data: {e}")
+            return []
             
             # Calculate number of chunks needed
             num_chunks = -(-total_length // effective_chunk_size)  # Ceiling division