Update app.py

app.py

@@ -1,14 +1,16 @@
+
 import os
 import logging
-import time
 from dotenv import load_dotenv
 import streamlit as st
 from PyPDF2 import PdfReader
 from langchain.text_splitter import CharacterTextSplitter
+# from langchain.embeddings import HuggingFaceInstructEmbeddings
 from langchain_cohere import CohereEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
+# from langchain.llms import Ollama
 from langchain_groq import ChatGroq
 
 # Load environment variables
@@ -40,34 +42,22 @@ def get_text_chunks(text):
     chunks = text_splitter.split_text(text)
     return chunks
 
-# Function to create a FAISS vectorstore with batching
+# Function to create a FAISS vectorstore
+# def get_vectorstore(text_chunks):
+#     embeddings = HuggingFaceInstructEmbeddings(model_name="hkunlp/instructor-xl")
+#     vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
+#     return vectorstore
+
 def get_vectorstore(text_chunks):
     cohere_api_key = os.getenv("COHERE_API_KEY")
     embeddings = CohereEmbeddings(model="embed-english-v3.0", cohere_api_key=cohere_api_key)
-    
-    # Batch processing to respect Cohere's rate limit
-    batch_size = 40
-    all_embeddings = []
-
-    for i in range(0, len(text_chunks), batch_size):
-        batch = text_chunks[i:i + batch_size]
-        logging.info(f"Processing batch {i // batch_size + 1}: {len(batch)} texts")
-        try:
-            batch_embeddings = embeddings.embed_documents(batch)
-            all_embeddings.extend(batch_embeddings)
-        except Exception as e:
-            logging.error(f"Error embedding batch {i // batch_size + 1}: {e}")
-            st.error(f"An error occurred while embedding batch {i // batch_size + 1}.")
-        if i + batch_size < len(text_chunks):  # Enforce delay only if more batches remain
-            logging.info("Waiting for 60 seconds to respect API rate limits...")
-            time.sleep(60)  # Wait for 60 seconds
-
-    vectorstore = FAISS.from_texts_with_embeddings(texts=text_chunks, embeddings=all_embeddings)
+    vectorstore = FAISS.from_texts(texts=text_chunks, embedding=embeddings)
     return vectorstore
 
 # Function to set up the conversational retrieval chain
 def get_conversation_chain(vectorstore):
     try:
+        # llm = Ollama(model="llama3.2:1b")
         llm = ChatGroq(model="llama-3.1-70b-versatile", temperature=0.5)
         memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)