update

processors/ngram_analysis.py

@@ -71,13 +71,24 @@ def compare_ngrams(texts, model_names, n=2, top_n=25):
         # Ensure all texts are strings and handle nested lists
         processed_texts = []
         for text in texts:
-            if isinstance(text, list):
-                # Flatten nested lists and join into a single string
-                flat_text = ' '.join(map(str, flatten_list(text)))
-                processed_texts.append(flat_text)
-            else:
-                # Convert non-string objects to strings
-                processed_texts.append(str(text))
+            try:
+                if isinstance(text, list):
+                    # More thoroughly flatten and ensure we have a string
+                    flat_items = list(flatten_list(text))
+                    # Convert each item to string and join
+                    flat_text = ' '.join([str(item) for item in flat_items])
+                    processed_texts.append(flat_text)
+                else:
+                    # Convert non-string objects to strings
+                    processed_texts.append(str(text))
+                    
+                # Verify we have a valid string
+                if not isinstance(processed_texts[-1], str):
+                    processed_texts[-1] = str(processed_texts[-1])
+            except Exception as e:
+                # Handle problematic text by adding empty string
+                print(f"Warning: Error processing text: {e}")
+                processed_texts.append("")
 
         X = vectorizer.fit_transform(processed_texts)