Update search_utils.py

search_utils.py

@@ -38,7 +38,6 @@ class MetadataManager:
         try:
             # Load the parquet file
             self.df = pd.read_parquet(self.metadata_path)
-            
             # Clean and format the data
             self.df['source'] = self.df['source'].apply(
                 lambda x: [
@@ -47,8 +46,12 @@ class MetadataManager:
                     if url.strip()
                 ]
             )
+            # Convert list of sources to a single string
+            self.df['source'] = self.df['source'].apply(lambda x: ' | '.join(x) if isinstance(x, list) else x)
+            
             self.total_docs = len(self.df)
             
+            
             logger.info(f"Successfully loaded {self.total_docs} documents")
         except Exception as e:
             logger.error(f"Failed to load metadata: {str(e)}")
@@ -238,59 +241,6 @@ class SemanticSearch:
             self.logger.error(f"Search failed in shard {shard_idx}: {str(e)}")
             return None
 
-    def _process_results(self, distances, global_indices, top_k):
-        """Process raw search results with correct similarity calculation for cosine similarity."""
-        process_start = time.time()
-        if global_indices.size == 0 or distances.size == 0:
-            self.logger.warning("No search results to process")
-            return pd.DataFrame(columns=["title", "summary", "source", "authors", "similarity"])
-    
-        try:
-            self.logger.info(f"Retrieving metadata for {len(global_indices)} indices")
-            metadata_start = time.time()
-            results = self.metadata_mgr.get_metadata(global_indices)
-            self.logger.info(f"Metadata retrieved in {time.time() - metadata_start:.2f}s, got {len(results)} records")
-    
-            if len(results) == 0:
-                self.logger.warning("No metadata found for indices")
-                return pd.DataFrame(columns=["title", "summary", "source", "authors", "similarity"])
-    
-            # Handle distance-results alignment
-            if len(results) != len(distances):
-                self.logger.warning(f"Mismatch between distances ({len(distances)}) and results ({len(results)})")
-                min_len = min(len(results), len(distances))
-                results = results.iloc[:min_len]
-                distances = distances[:min_len]
-
-            # For inner product with normalized vectors, similarity is directly the distance
-            # (FAISS IP search already returns higher scores for more similar items)
-            results['similarity'] = 1 - (distances/2)
-            
-            # Deduplicate and sort
-            required_columns = ["title", "summary", "authors", "source", "similarity"]
-            pre_dedup = len(results)
-            results = (
-                results.drop_duplicates(subset=["title", "authors"])
-                .sort_values("similarity", ascending=False)
-                .head(top_k)
-            )
-            post_dedup = len(results)
-            
-            if pre_dedup > post_dedup:
-                self.logger.info(f"Removed {pre_dedup - post_dedup} duplicate results")
-
-            self.logger.info(f"Results processed in {time.time() - process_start:.2f}s")
-            return results[required_columns].reset_index(drop=True)
-            
-        except Exception as e:
-            self.logger.error(f"Result processing failed: {str(e)}", exc_info=True)
-            return pd.DataFrame(columns=["title", "summary", "source", "authors", "similarity"])
-    
-
-    
-
-    
-
     def _process_results(self, distances, global_indices, top_k):
         """Process raw search results into formatted DataFrame"""
         process_start = time.time()
@@ -337,7 +287,7 @@ class SemanticSearch:
             
             # Deduplicate and sort results
             pre_dedup = len(results)
-            results = results.drop_duplicates(subset=["title", "source"]).sort_values("similarity", ascending=False).head(top_k)
+            results = results.drop_duplicates(subset=["title").sort_values("similarity", ascending=False).head(top_k)
             post_dedup = len(results)
             
             if pre_dedup > post_dedup:
@@ -359,4 +309,4 @@ class SemanticSearch:
             
         except Exception as e:
             self.logger.error(f"Result processing failed: {str(e)}", exc_info=True)
-            return pd.DataFrame(columns=["title", "summary", "similarity"])
+            return pd.DataFrame(columns=["title", "summary", "similarity", 'authors'])