duplicate issue

app.py

@@ -7,6 +7,8 @@ import torch
 from functools import lru_cache
 import logging
 from datetime import datetime
+from collections import defaultdict
+
 
 # 🔧 Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -70,12 +72,23 @@ def encode_splade_cached(text: str) -> SparseVector:
     vocab_indices = nonzero[:, 1]
     values = relu_log[nonzero[:, 0], nonzero[:, 1]]
 
-    logger.info(f"SPLADE encoding complete with {len(vocab_indices)} dimensions")
+    vocab_indices_list = vocab_indices.cpu().numpy().tolist()
+    values_list = values.cpu().numpy().tolist()
+
+    index_to_value = defaultdict(float)
+    for idx, val in zip(vocab_indices_list, values_list):
+        index_to_value[idx] += val
+
+    deduped_indices = list(index_to_value.keys())
+    deduped_values = list(index_to_value.values())
+
+    logger.info(f"SPLADE encoding complete with {len(deduped_indices)} dimensions")
     return SparseVector(
-        indices=vocab_indices.cpu().numpy().tolist(),
-        values=values.cpu().numpy().tolist()
+        indices=deduped_indices,
+        values=deduped_values
     )
 
+
 # 🚀 Main endpoint
 @app.post("/get-embedding/")
 async def get_embedding(input: TextInput):