added pinecone for project

src/backend/data/dataset.py

@@ -9,29 +9,33 @@ DATASET_PATH = "src/backend/data/dataset.csv"
 PARAQUET_DATASET_PATH = "hf://datasets/lavita/ChatDoctor-HealthCareMagic-100k/data/train-00000-of-00001-5e7cb295b9cff0bf.parquet"
 
 def get_data_set():
-
     try:
         if not os.path.exists(DATASET_PATH):
             logger.info(f"{DATASET_PATH} not found. Reading from Parquet file.")
             df = pd.read_parquet(PARAQUET_DATASET_PATH)
-            df.drop_duplicates(subset=["input", "output"], inplace=True)
-            df.dropna(subset=["input", "output"], inplace=True) # Remove NaNs first
-        
-        # This line is to remove the empty column or column with only spaces
-            df = df[(df["input"].str.strip() != "") & (df["output"].str.strip() != "")] # Remove empty strings/spaces
-
-        # This line is to remove puncuation and emjois
-            translator = str.maketrans('', '', string.punctuation)
-            df["input"] = df["input"].str.lower().str.translate(translator)
-            df["output"] = df["output"].str.lower().str.translate(translator)
-            df.to_csv(DATASET_PATH, index=False)
-            logger.info(f"CSV file created and cleaned at: {DATASET_PATH}")
         else:
             logger.info(f"Loading existing dataset from: {DATASET_PATH}")
-            df = pd.read_csv(DATASET_PATH)
-            logger.info("Dataset loaded successfully.")
+            df = pd.read_csv(DATASET_PATH).fillna("")
+
+        # Cleaning logic for both Parquet and CSV data
+        df.drop_duplicates(subset=["input", "output"], inplace=True)
+
+        # Remove NaN values or empty strings
+        df = df[df["input"].str.strip().notna() & df["output"].str.strip().notna()]
+        df = df[(df["input"].str.strip() != "") & (df["output"].str.strip() != "")]
+
+        # Clean punctuation and emojis
+        translator = str.maketrans('', '', string.punctuation)
+        df["input"] = df["input"].fillna("").str.lower().str.translate(translator)
+        df["output"] = df["output"].fillna("").str.lower().str.translate(translator)
+
+        # Save only if data is present
+        if not os.path.exists(DATASET_PATH):
+            df.to_csv(DATASET_PATH, index=False)
+            logger.info(f"CSV file created and cleaned at: {DATASET_PATH}")
+
         return df
 
     except Exception as e:
         logger.error(f"Error while loading dataset: {e}", exc_info=True)
-        return None
+        return None

src/backend/data/pinecone_db.py

@@ -61,37 +61,6 @@ def get_index():
 
 index = get_index()
 
-def process_and_upsert_data(index, data: pd.DataFrame):
-
-    # Validate if the required columns exist in the row (Series)
-    try:
-        logger.info("Started upserting the data to database")
-        for idx, row in data.iterrows():
-            logger.info(f"Processing row {row['input']}")
-            input_text = row['input']
-            output_text = row['output']
-            instruction_text = row['instruction']
-            if not isinstance(input_text, str) or not input_text.strip():
-                logger.warning(f"Skipping row {idx} due to empty or invalid input text.")
-                continue
-            row_dict = {
-                "question": input_text,
-                "answer" : output_text,
-                "instruction": instruction_text
-            }
-            embeddings = embedding_model.get_text_embedding(row['input'])
-            index.upsert(
-            vectors=[{
-                "id": f"id{idx}",
-                "values": embeddings,
-                "metadata":row_dict
-            }],
-            namespace=NAMESPACE,
-        )
-        logger.info(f"Successfully upserted data for question {input_text} with answer {output_text}")
-    except Exception as e:
-        logger.error(f"Error processing row with index {idx}: {e}")
-
 def search_vector_store(query, n_result : int = 3) -> list[dict]:
     """
     Searches the vector store for the most relevant matches based on the given query.
@@ -138,27 +107,51 @@ def get_retrieved_context(prompt: str) -> str:
         return "\n".join(retrieved_contexts[:3])
     return "No relevant information found in the database."
 
-df = dataset.get_data_set()[6:200]
-# process_and_upsert_data(index, data_set)
-# response = search_vector_store("What is the treatment for diabetes?")
-# print(response)
+def upsert_data_in_db(df: pd.DataFrame):
 
+    """
+    Generates embeddings for the given DataFrame and uploads data to Pinecone in batches.
+    
+    Parameters:
+    - df (pd.DataFrame): DataFrame containing 'input', 'question', and 'answer' columns.
+    
+    Returns:
+    - None
+    """
 
-def upsert_data_in_db(df: pd.DataFrame):
-    df["embedding"] = [embedding_model.get_text_embedding([q])[0] for q in tqdm(df["input"], desc="Embedding Questions")]
+    try:
+        df["embedding"] = [
+            embedding_model.get_text_embedding([q])[0] 
+            for q in tqdm(df["input"], desc="Generating Embeddings")
+        ]
+    except Exception as e:
+        logger.error(f"Error generating embeddings: {e}")
+        return
 
-    # Upload data to Pinecone in batches
-    BATCH_SIZE = 100
-    vectors = []
+    # # Upload data to Pinecone in batches
+    BATCH_SIZE = 500
 
-    for i in tqdm(range(0, len(df), BATCH_SIZE), desc="Storing Data in Pinecone"):
+    for i in tqdm(range(0, len(df), BATCH_SIZE), desc="Uploading Data to Pinecone"):
         batch = df.iloc[i : i + BATCH_SIZE]
-        vectors = [
-            (f"q_{idx}", emb, {"question": row[0], "answer": row[1], "instruction": row[2]}) 
-            for idx, (emb, row) in enumerate(zip(batch["embedding"], batch.iterrows()))
-        ]
-        index.upsert(vectors)  # Upsert (insert or update) in Pinecone
+    
+        vectors = []
+        for idx, (embedding, (_, row_data)) in enumerate(zip(batch["embedding"], batch.iterrows())):
+            vector_id = f"q_{i + idx}"  # Ensures IDs remain unique across batches
+            metadata = {
+                "question": row_data.get("input"),
+                "answer": row_data.get("output")
+            }
+            vectors.append((vector_id, embedding, metadata))
+
+        try:
+            index.upsert(vectors)
+        except Exception as e:
+            logger.error(f"Error uploading batch starting at index {i}: {e}")
 
-    print("✅ All question-answer pairs stored successfully!")
+    logger.info("All question-answer pairs stored successfully!")
 
-upsert_data_in_db(df)
+
+# df = dataset.get_data_set()[19000:21000]
+# upsert_data_in_db(df)
+# response = search_vector_store("What is the treatment for diabetes?")
+# print(response)

src/backend/main.py

@@ -1,7 +1,7 @@
 from fastapi import FastAPI
-from routes import chat_api
+from routes import chat_api,upsert_data
 
 app = FastAPI()
 
 app.include_router(chat_api.router, prefix="/chat", tags=["chat"])
-# app.include_router(upsert_data.router, prefix="/data", tags=["data"])
+app.include_router(upsert_data.router, prefix="/data", tags=["data"])

src/backend/models/embedding_model.py

@@ -9,9 +9,8 @@ model = SentenceTransformer("all-MiniLM-L6-v2")
 
 def get_text_embedding(search_query: str):
     try:
-        logger.info(f"Getting embedding for the text: {search_query}")
         text_embedding = model.encode(search_query, convert_to_tensor=True).cpu().numpy().tolist()
-        logger.info("Text embedding successfully retrieved.")
+        # logger.info("Text embedding successfully retrieved.")
         return text_embedding
     except Exception as e:
         logger.error(f"Error while getting embedding for text: {e}")

src/backend/models/llm_model.py

@@ -2,6 +2,7 @@ import os
 from groq import Groq
 from utils import logger
 from data import chroma_db
+from data import pinecone_db
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -25,7 +26,7 @@ def get_medical_assistant_response(prompt: list):
         if not prompt or len(prompt[0]) < 5:
             return "⚠️ Your question seems too short. Please provide more details so I can assist you better."
         query = prompt[-1]
-        response = chroma_db.search_vector_store(query)
+        response = pinecone_db.search_vector_store(query)
         
         if response and "metadatas" in response and response["metadatas"]:
             retrieved_contexts = [metadata['answer'] for metadata in response["metadatas"][0]]
@@ -68,7 +69,7 @@ def get_medical_assistant_request(conversation_history: list):
             return "⚠️ Please provide more details so I can assist you better."
         latest_user_message = conversation_history[-1]["content"]
         retrieved_contexts = []
-        chroma_response = chroma_db.search_vector_store(latest_user_message)
+        chroma_response = pinecone_db.search_vector_store(latest_user_message)
         if chroma_response and "metadatas" in chroma_response and chroma_response["metadatas"]:
             retrieved_contexts = [metadata['answer'] for metadata in chroma_response["metadatas"][0]]
         context = "\n".join(retrieved_contexts[:3]) if retrieved_contexts else "No relevant information found in the database."

src/backend/models/schemas.py

@@ -6,4 +6,8 @@ class Chat_Response(BaseModel):
     response: Optional[Dict] = None
 
 class ChatRequest(BaseModel):
-    conversation_history: List[Dict]
+    conversation_history: List[Dict]
+
+class Add_Data_In_DB(BaseModel):
+    start: int
+    end: int

src/backend/routes/upsert_data.py

@@ -1,15 +1,19 @@
-# from fastapi import APIRouter,HTTPException
-# from data import dataset
-# from data import pinecone_db
+from fastapi import APIRouter,HTTPException
+from data import dataset
+from data import pinecone_db
+from models.schemas import Add_Data_In_DB
 
-# router = APIRouter()
-# index_name = "question-answering-index"
+router = APIRouter()
+index_name = "question-answering-index"
 
-# @router.post("/upsert_data")
-# async def upsert_data():
-#     try:
-#         df = dataset.get_data_set()[0:1000]
-#         pinecone_db.process_and_upsert_data(index_name, df)
-#         return {"status": "success"}
-#     except Exception as e:
-#         raise HTTPException(status_code=500, detail=str(e))
+@router.post("/upsert_data")
+async def upsert_data(add_data: Add_Data_In_DB):
+
+    try:
+        start = add_data.start
+        end = add_data.end
+        df = dataset.get_data_set()[start:end]
+        pinecone_db.upsert_data_in_db(df, index_name)
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

src/frontend/app/common_fuctions.py

@@ -3,6 +3,7 @@ import base64
 import requests
 from dotenv import load_dotenv
 from utils import logger
+import json
 
 load_dotenv()
 logger = logger.get_logger()
@@ -27,12 +28,29 @@ def get_api_response(endpoint:str, prompt: list):
         logger.info(f"Sending user prompt to API endpoint: {API_URL}{endpoint}")
         response = requests.post(f"{API_URL}{endpoint}", json={"prompt": prompt})
         if response.status_code == 200:
-            return response.json()["response"]
+            return response.json()
         else:
             return "An error occurred while processing your request."
     except Exception as e:
         return f"An error occurred while processing your request: {str(e)}"
     
+
+def upsert_data_request(start, end):
+    headers = {"Content-Type": "application/json"}
+    payload = {
+        "start": start,
+        "end": end
+    }
+
+    try:
+        url = "http://localhost:8000/data/upsert_data"
+        response = requests.post(url, data=json.dumps(payload), headers=headers)
+        return response
+    except requests.exceptions.HTTPError as http_err:
+        print(f"HTTP error occurred: {http_err}")
+    except Exception as err:
+        print(f"An error occurred: {err}")
+    
 def initialize_conversation():
 
     assistant_message = "Hello! I am Yuvabe Care Companion AI. How can I assist you with your health-related queries today?"

src/frontend/app/homepage.py

@@ -60,11 +60,11 @@ def handle_user_input():
             response = "⚠️ Oops! Something went wrong. Please try again."
 
         with st.chat_message("assistant"):
-            st.markdown(response)
+            st.markdown(response['response'])
 
         st.session_state.messages.append({"role": "assistant", "content": response})
 
-        logger.info(f"Assistant response: {response[:100]}...")
+        logger.info(f"Assistant response: {response['response'][:100]}...")
 
 # def handle_user_input():
 

src/frontend/pages/database_response_page.py

@@ -19,10 +19,9 @@ if prompt:
         endpoint = "/chat/db_response"
         response = common_fuctions.get_api_response(endpoint, [prompt])
         st.subheader("✅ Relevant question and answer pair found in the database.")
-        for metadata_group in response["metadatas"]:
-            for entry in metadata_group:
-                st.write("Question:", entry["question"])
-                st.write("Answer:", entry["answer"])
+        for metadata in response:
+                st.write("Question:", metadata["question"])
+                st.write("Answer:", metadata["answer"])
                 st.write("-" * 80) 
 
         if st.button("Clear chat"):

src/frontend/pages/dataloader_page.py

@@ -0,0 +1,43 @@
+from app import common_fuctions
+import streamlit as st
+from app import homepage
+from utils import logger
+
+logger = logger.get_logger()
+
+homepage.config_homepage()
+st.title("Data Loader")
+
+def load_data():
+    st.sidebar.header("📊 Data Loading Parameters")
+    start_index  = st.sidebar.number_input("Select start index", min_value=0, value=0)
+    end_index  = st.sidebar.number_input("Select end index", min_value=0, value=100)
+
+    if start_index > end_index:
+        st.sidebar.error("⚠️ Start index must be earlier than the end index.")
+        return
+    
+    if "load_clicked" not in st.session_state:
+        st.session_state.load_clicked = False
+
+    try:
+        st.sidebar.info(f"Click the button to load data from index **{start_index} to {end_index}**.")
+        if st.sidebar.button("🚀 Upsert Data", disabled=st.session_state.load_clicked, help="Click to insert data into the database"):
+            st.session_state.load_clicked = True
+
+            with st.spinner("⏳ Upserting data... Please wait"):
+                response = common_fuctions.upsert_data_request(start_index, end_index)
+                st.write(response)
+                # if response.get("status") == "success":
+                #     st.success("Data upserted successfully!")
+                #     st.session_state.load_clicked = False
+                # else:
+                #     st.error("Failed to upsert data.")
+                #     logger.error("Failed to upsert data.")
+                #     st.session_state.load_clicked = False
+    except Exception as e:
+        st.error(f"Error loading data: {e}")
+        logger.error(f"Error loading data: {e}")
+        st.session_state.load_clicked = False
+
+load_data()