added pinecone db

chroma.log

@@ -0,0 +1,20 @@
+WARNING:  [10-03-2025 20:07:32] chroma_server_nofile is not supported on Windows. chroma_server_nofile will not be set.
+WARNING:  [10-03-2025 20:07:51] chroma_server_nofile is not supported on Windows. chroma_server_nofile will not be set.
+INFO:     [10-03-2025 20:07:52] Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.
+DEBUG:    [10-03-2025 20:07:52] Starting component System
+DEBUG:    [10-03-2025 20:07:52] Starting component OpenTelemetryClient
+DEBUG:    [10-03-2025 20:07:52] Starting component SqliteDB
+DEBUG:    [10-03-2025 20:07:52] Starting component SimpleQuotaEnforcer
+DEBUG:    [10-03-2025 20:07:52] Starting component Posthog
+DEBUG:    [10-03-2025 20:07:52] Starting component SimpleRateLimitEnforcer
+DEBUG:    [10-03-2025 20:07:52] Starting component LocalSegmentManager
+DEBUG:    [10-03-2025 20:07:52] Starting component LocalExecutor
+DEBUG:    [10-03-2025 20:07:52] Starting component SegmentAPI
+DEBUG:    [10-03-2025 20:07:52] Starting component SimpleAsyncRateLimitEnforcer
+INFO:     [10-03-2025 20:07:52] Started server process [17592]
+INFO:     [10-03-2025 20:07:52] Waiting for application startup.
+INFO:     [10-03-2025 20:07:52] Application startup complete.
+INFO:     [10-03-2025 20:07:52] Uvicorn running on http://localhost:8000 (Press CTRL+C to quit)
+INFO:     [10-03-2025 20:16:05] ::1:57148 - "GET /api/v2/auth/identity HTTP/1.1" 200
+INFO:     [10-03-2025 20:16:05] ::1:57149 - "GET /api/v2/tenants/default_tenant HTTP/1.1" 200
+INFO:     [10-03-2025 20:16:05] ::1:57149 - "GET /api/v2/tenants/default_tenant/databases/default_database HTTP/1.1" 200

src/backend/data/chroma_db.py

@@ -1,4 +1,5 @@
 import chromadb
+import asyncio
 from utils import logger
 from chromadb.utils import embedding_functions
 default_ef = embedding_functions.DefaultEmbeddingFunction()
@@ -12,6 +13,11 @@ DB_PATH = "./src/backend/vector-db"
 # Initialize ChromaDB Client
 client = chromadb.PersistentClient(path=DB_PATH)
 
+
+
+# chroma_client = chromadb.HttpClient(host='localhost', port=8000)
+# client = chromadb.AsyncHttpClient()
+
 collection = client.get_or_create_collection(
     name=COLLECTION_NAME,
     embedding_function=default_ef,

src/backend/data/pinecone_db.py

@@ -0,0 +1,144 @@
+import os
+import sys
+src_directory = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..", "backend"))
+sys.path.append(src_directory)
+from pinecone import Pinecone, ServerlessSpec
+import time
+from tqdm import tqdm  # Progress bar for large datasets
+from dotenv import load_dotenv
+from utils import logger
+import pandas as pd
+from models import embedding_model
+from data import dataset
+
+load_dotenv()
+PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY")
+logger = logger.get_logger()
+NAMESPACE = "health-care-dataset"
+INDEX_NAME = "health-care-index"
+
+
+def create_index(pinecone, index_name):
+    pinecone.create_index(
+    name=index_name,
+    dimension=384,
+    metric="cosine",
+    spec=ServerlessSpec(
+    cloud="aws",
+    region="us-east-1"
+        ) 
+    )
+
+def wait_till_index_loaded(pinecone, index_name):
+    while True:
+        index = pinecone.describe_index(index_name)
+        if index.status.get("ready", False):
+            index = pinecone.Index(index_name)
+            logger.info(f"Index '{index_name}' is ready and is now accessible.")
+            return index
+        else:
+            logger.debug(f"Index '{index_name}' is not ready yet. Checking again in 1 second.")
+            time.sleep(1)
+
+def get_index():
+    global index
+    index = None
+    try:
+        pc = Pinecone(api_key=PINECONE_API_KEY)
+        index_name = INDEX_NAME
+        logger.info(f"Checking if the index '{index_name}' exists...")
+        if not pc.has_index(index_name):
+            logger.info(f"Index '{index_name}' does not exist. Creating a new index...")
+            create_index(pc,index_name)
+            logger.info(f"Index '{index_name}' creation initiated. Waiting for it to be ready...")
+            index = wait_till_index_loaded(pc,index_name)
+        else:
+            index = pc.Index(index_name)
+            logger.info(f"Index '{index_name}' already exists. Returning the existing index.")
+    except Exception as e:
+        logger.info(f"Error occurred while getting or creating the Pinecone index: {str(e)}", exc_info=True)
+    return index
+
+index = get_index()
+
+def process_and_upsert_data(index, data: pd.DataFrame):
+
+    # Validate if the required columns exist in the row (Series)
+    try:
+        logger.info("Started upserting the data to database")
+        for idx, row in data.iterrows():
+            logger.info(f"Processing row {row['input']}")
+            input_text = row['input']
+            output_text = row['output']
+            instruction_text = row['instruction']
+            if not isinstance(input_text, str) or not input_text.strip():
+                logger.warning(f"Skipping row {idx} due to empty or invalid input text.")
+                continue
+            row_dict = {
+                "question": input_text,
+                "answer" : output_text,
+                "instruction": instruction_text
+            }
+            embeddings = embedding_model.get_text_embedding(row['input'])
+            index.upsert(
+            vectors=[{
+                "id": f"id{idx}",
+                "values": embeddings,
+                "metadata":row_dict
+            }],
+            namespace=NAMESPACE,
+        )
+        logger.info(f"Successfully upserted data for question {input_text} with answer {output_text}")
+    except Exception as e:
+        logger.error(f"Error processing row with index {idx}: {e}")
+
+def search_vector_store(query, n_result : int = 3) -> list[dict]:
+    """
+    Searches the vector store for the most relevant matches based on the given query.
+
+    This method retrieves the top `n_result` closest matches from the vector store
+    using an embedding-based similarity search. Each match includes metadata 
+    such as the answer, instruction, and question.
+
+    Args:
+        query (str): The search query text.
+        n_result (int, optional): The number of top results to retrieve. Defaults to 3.
+
+    Returns:
+        list[dict]: A list of dictionaries, where each dictionary contains:
+            - "answer" (str): The retrieved answer.
+            - "instruction" (str): The instruction related to the answer.
+            - "question" (str): The question associated with the answer.
+
+    Raises:
+        Exception: If an error occurs while querying the vector store.
+
+    """
+    try:
+        index = get_index()
+        embedding = embedding_model.get_text_embedding(query)
+        response = index.query(
+            top_k=n_result,
+            vector=embedding,
+            namespace=NAMESPACE,
+            include_metadata=True)
+        metadata = []
+        for response in response['matches']:
+            metadata.append({"answer":response['metadata']['answer'], 
+                        "instruction":response['metadata']['instruction'], 
+                        "question":response['metadata']['question']})
+        return metadata
+    except Exception as e:
+        raise Exception(f"Error occurred while searching the vector store: {str(e)}")
+
+def get_retrieved_context(prompt: str) -> str:
+    response = search_vector_store(prompt)
+    if response and "metadatas" in response and response["metadatas"]:
+        retrieved_contexts = [metadata["answer"] for metadata in response["metadatas"][0]]
+        return "\n".join(retrieved_contexts[:3])
+    return "No relevant information found in the database."
+
+data_set = dataset.get_data_set()[6139:10000]
+process_and_upsert_data(index, data_set)
+# response = search_vector_store("What is the treatment for diabetes?")
+# print(response)

src/backend/main.py

@@ -3,4 +3,5 @@ from routes import chat_api
 
 app = FastAPI()
 
-app.include_router(chat_api.router, prefix="/chat", tags=["chat"])
+app.include_router(chat_api.router, prefix="/chat", tags=["chat"])
+# app.include_router(upsert_data.router, prefix="/data", tags=["data"])

src/backend/models/embedding_model.py

@@ -0,0 +1,18 @@
+from sentence_transformers import SentenceTransformer
+from utils import logger
+from fastapi import UploadFile
+from PIL import Image
+
+logger = logger.get_logger()
+
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+def get_text_embedding(search_query: str):
+    try:
+        logger.info(f"Getting embedding for the text: {search_query}")
+        text_embedding = model.encode(search_query, convert_to_tensor=True).cpu().numpy().tolist()
+        logger.info("Text embedding successfully retrieved.")
+        return text_embedding
+    except Exception as e:
+        logger.error(f"Error while getting embedding for text: {e}")
+        raise

src/backend/routes/chat_api.py

@@ -2,7 +2,8 @@ from fastapi import APIRouter, HTTPException
 from utils import logger
 from models.schemas import Chat_Response, ChatRequest
 from models import llm_model
-from data import chroma_db
+from data import pinecone_db
+# from data import chroma_db
 
 logger = logger.get_logger()
 router = APIRouter()
@@ -23,9 +24,9 @@ async def get_db_response(chat_request: Chat_Response):
     try:
         logger.info(f"Received user prompt: {chat_request.prompt}")
         query = chat_request.prompt[-1]
-        response_text  = chroma_db.search_vector_store(query)
+        response_text  = pinecone_db.search_vector_store(query)
         logger.info(f"Retrieved context for user prompt: {chat_request.prompt[:50]}...")
-        return {"status": "success", "response": response_text}
+        return response_text
     except Exception as e:
         logger.exception("Unexpected error occurred while processing the request.")
         raise HTTPException(status_code=500, detail="An error occurred while processing your request.")
@@ -37,4 +38,6 @@ async def chat_with_assistant(request: ChatRequest):
         response = llm_model.get_medical_assistant_request(request.conversation_history)
         return {"response": response}
     except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        raise HTTPException(status_code=500, detail=str(e))
+    
+# Input format for above endpoint

src/backend/routes/upsert_data.py

@@ -0,0 +1,15 @@
+# from fastapi import APIRouter,HTTPException
+# from data import dataset
+# from data import pinecone_db
+
+# router = APIRouter()
+# index_name = "question-answering-index"
+
+# @router.post("/upsert_data")
+# async def upsert_data():
+#     try:
+#         df = dataset.get_data_set()[0:1000]
+#         pinecone_db.process_and_upsert_data(index_name, df)
+#         return {"status": "success"}
+#     except Exception as e:
+#         raise HTTPException(status_code=500, detail=str(e))