Update index_miriad_to_qdrant.py

index_miriad_to_qdrant.py

@@ -16,7 +16,7 @@ client = QdrantClient(
 )
 
 # Load MIRIAD dataset (sample for demo)
-ds = load_dataset("miriad/miriad-5.8M", split="train").select(range(100000))
+ds = load_dataset("miriad/miriad-5.8M", split="train").select(range(1000))
 
 dense_documents = [
     models.Document(text=doc, model="BAAI/bge-small-en")
@@ -32,7 +32,7 @@ collection_name = "medical_chat_bot"
 
 # Create collection
 if not client.collection_exists(collection_name):
-    client.recreate_collection(
+    client.create_collection(
         collection_name=collection_name,
         vectors_config={
             "dense": models.VectorParams(size=384, distance=models.Distance.COSINE),
@@ -47,6 +47,19 @@ if not client.collection_exists(collection_name):
         }
     )
 
+    # Create payload indexes
+    client.create_payload_index(
+        collection_name=collection_name,
+        field_name="specialty",
+        field_schema="keyword",
+    )
+
+    client.create_payload_index(
+        collection_name=collection_name,
+        field_name="year",
+        field_schema="integer",
+    )
+
 # Batch upload in chunks
 BATCH_SIZE = 3
 points_batch = []
@@ -58,7 +71,11 @@ for i in range(len(ds['passage_text'])):
             "dense": dense_documents[i],
             "colbert": colbert_documents[i]
         },
-        payload={"passage_text": ds['passage_text'][i], "paper_id": ds['paper_id'][i]}
+        payload={
+        "passage_text": ds['passage_text'][i],
+         "year": ds['year'][i],
+         "specialty": ds['specialty'][i],
+         }
     )
     points_batch.append(point)