Update app.py

app.py

@@ -1,122 +1,132 @@
-# importing dependencies
+import os
 from dotenv import load_dotenv
 import streamlit as st
-from PyPDF2 import PdfReader
 from langchain.text_splitter import CharacterTextSplitter
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import faiss
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
 from langchain.prompts import PromptTemplate
 from langchain.memory import ConversationBufferMemory
 from langchain.chains import ConversationalRetrievalChain
 from langchain.chat_models import ChatOpenAI
-from htmlTemplates import css, bot_template, user_template
-from langchain.embeddings import openai
-from langchain.embeddings.openai import OpenAIEmbeddings
-import os
+from langchain.document_loaders import PyPDFLoader, Docx2txtLoader, TextLoader, CSVLoader
+
+# Load environment variables from .env
+load_dotenv()
+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")
 
- 
-from openai import OpenAI
-api_key = os.getenv("OPENAI_API_KEY")
-client = OpenAI(api_key=api_key)
-
-
-# creating custom template to guide llm model
-custom_template ="""<s>[INST]You will start the conversation by greeting the user and introducing yourself as qanoon-bot,\
-stating your availability for legal assistance. Your next step will depend on the user's response.\
-If the user expresses a need for legal assistance in Pakistan, you will ask them to describe their case or problem.\
-After receiving the case or problem details from the user, you will provide the solutions and procedures according to the knowledge base and also give related penal codes and procedures. \
-However, if the user does not require legal assistance in Pakistan, you will immediately thank them and\
-say goodbye, ending the conversation. Remember to base your responses on the user's needs, providing accurate and\
-concise information regarding the Pakistan legal law and rights where applicable. Your interactions should be professional and\
-focused, ensuring the user's queries are addressed efficiently without deviating from the set flows.\
+# Custom template to guide the LLM
+custom_template = """
+<s>[INST]You will start the conversation by greeting the user and introducing yourself as an Expert PDF documents analyzer and assistant,
+stating your availability for assistance. Your next step will depend on the user's response.
+If the user expresses a need for assistance in pdf or document, you will ask them to describe their question.
+However, if the user asks questions out of context from the knowledge base, you will immediately thank them and
+say goodbye, ending the conversation. Remember to base your responses on the user's needs, providing accurate and
+concise information regarding the data within the knowledge base. Your interactions should be professional and
+focused, ensuring the user's queries are addressed efficiently without deviating from the set flows.
 CHAT HISTORY: {chat_history}
 QUESTION: {question}
 ANSWER:
 </s>[INST]
 """
-
 CUSTOM_QUESTION_PROMPT = PromptTemplate.from_template(custom_template)
 
-# extracting text from pdf
-def get_pdf_text(docs):
-    text=""
-    for pdf in docs:
-        pdf_reader=PdfReader(pdf)
-        for page in pdf_reader.pages:
-            text+=page.extract_text()
-    return text
-
-# converting text to chunks
-def get_chunks(raw_text):
-    text_splitter=CharacterTextSplitter(separator="\n",
-                                        chunk_size=1000,
-                                        chunk_overlap=200,
-                                        length_function=len)   
-    chunks=text_splitter.split_text(raw_text)
+# Convert text to chunks
+def get_chunks(documents):
+    text_splitter = CharacterTextSplitter(separator="\n", chunk_size=1000, chunk_overlap=200, length_function=len)
+    chunks = [chunk for doc in documents for chunk in text_splitter.split_text(doc.page_content)]
     return chunks
 
-# using all-MiniLm embeddings model and faiss to get vectorstore
+# Create vectorstore using OpenAI embeddings and FAISS
 def get_vectorstore(chunks):
-    embeddings=OpenAIEmbeddings()
-    vectorstore=faiss.FAISS.from_texts(texts=chunks,embedding=embeddings)
+    embeddings = OpenAIEmbeddings()
+    vectorstore = FAISS.from_texts(texts=chunks, embedding=embeddings)
     return vectorstore
 
-# generating conversation chain  
+# Create conversation chain for LLM interaction
 def get_conversationchain(vectorstore):
-    llm=ChatOpenAI(temperature=0.4,model_name='gpt-4o-mini')
-    memory = ConversationBufferMemory(memory_key='chat_history', 
-                                      return_messages=True,
-                                      output_key='answer') # using conversation buffer memory to hold past information
+    llm = ChatOpenAI(temperature=0.4, model_name='gpt-4')
+    memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)
     conversation_chain = ConversationalRetrievalChain.from_llm(
-                                llm=llm,
-                                retriever=vectorstore.as_retriever(),
-                                condense_question_prompt=CUSTOM_QUESTION_PROMPT,
-                                memory=memory)
+        llm=llm,
+        retriever=vectorstore.as_retriever(),
+        condense_question_prompt=CUSTOM_QUESTION_PROMPT,
+        memory=memory
+    )
     return conversation_chain
 
-# generating response from user queries and displaying them accordingly
-def handle_question(question):
-    response=st.session_state.conversation({'question': question})
-    st.session_state.chat_history=response["chat_history"]
-    for i,msg in enumerate(st.session_state.chat_history):
-        if i%2==0:
-            st.write(user_template.replace("{{MSG}}",msg.content,),unsafe_allow_html=True)
-        else:
-            st.write(bot_template.replace("{{MSG}}",msg.content),unsafe_allow_html=True)
+# Extract text from various document types including PDFs, TXT, DOCX, and CSV.
+def get_document_text(uploaded_files):
+    documents = []
+
+    for uploaded_file in uploaded_files:
+        file_extension = os.path.splitext(uploaded_file.name)[1].lower()
+
+        if file_extension == ".pdf":
+            loader = PyPDFLoader(uploaded_file)
+            documents.extend(loader.load())
+        elif file_extension in [".docx", ".doc"]:
+            loader = Docx2txtLoader(uploaded_file)
+            documents.extend(loader.load())
+        elif file_extension == ".txt":
+            loader = TextLoader(uploaded_file)
+            documents.extend(loader.load())
+        elif file_extension == ".csv":
+            loader = CSVLoader(uploaded_file)
+            documents.extend(loader.load())
+
+    return documents
 
+# Function to process and handle a user's query
+def handle_question(conversation_chain, question):
+    response = conversation_chain({'question': question})
+    return response['answer']
 
+# Streamlit app
 def main():
-    load_dotenv()
-    st.set_page_config(page_title="Chat with multiple PDFs",page_icon=":books:")
-    st.write(css,unsafe_allow_html=True)
-    if "conversation" not in st.session_state:
-        st.session_state.conversation=None
-
-    if "chat_history" not in st.session_state:
-        st.session_state.chat_history=None
-    
-    st.header("Chat with multiple PDFs :books:")
-    question=st.text_input("Ask question from your document:")
-    if question:
-        handle_question(question)
-    with st.sidebar:
-        st.subheader("Your documents")
-        docs=st.file_uploader("Upload your PDF here and click on 'Process'",accept_multiple_files=True)
-        if st.button("Process"):
-            with st.spinner("Processing"):
-                
-                #get the pdf
-                raw_text=get_pdf_text(docs)
-                
-                #get the text chunks
-                text_chunks=get_chunks(raw_text)
-                
-                #create vectorstore
-                vectorstore=get_vectorstore(text_chunks)
-                
-                #create conversation chain
-                st.session_state.conversation=get_conversationchain(vectorstore)
+    st.set_page_config(page_title="Chat with Documents", page_icon=":books:")
+    st.title("Chat with Your Documents :books:")
+
+    # Session state for conversation and chat history
+    if "conversation_chain" not in st.session_state:
+        st.session_state.conversation_chain = None
+
+    st.sidebar.header("Upload Your Documents")
+    uploaded_files = st.sidebar.file_uploader(
+        "Upload your documents here (PDF, TXT, DOCX, CSV):",
+        type=["pdf", "txt", "docx", "csv"],
+        accept_multiple_files=True
+    )
+
+    if st.sidebar.button("Process"):
+        if uploaded_files:
+            with st.spinner("Processing your documents..."):
+                # Extract text from uploaded documents
+                raw_documents = get_document_text(uploaded_files)
+
+                if not raw_documents:
+                    st.error("No text could be extracted from the documents. Please check the files.")
+                    return
+
+                # Convert text to chunks
+                text_chunks = get_chunks(raw_documents)
+
+                # Create vectorstore
+                vectorstore = get_vectorstore(text_chunks)
+
+                # Create conversation chain
+                st.session_state.conversation_chain = get_conversationchain(vectorstore)
+
+                st.success("Documents processed successfully! You can now ask questions.")
+        else:
+            st.error("Please upload at least one document.")
 
+    # Chat interface
+    if st.session_state.conversation_chain:
+        question = st.text_input("Ask a question about your documents:")
+        if question:
+            with st.spinner("Generating response..."):
+                answer = handle_question(st.session_state.conversation_chain, question)
+                st.markdown(f"**Answer:** {answer}")
 
 if __name__ == '__main__':
-    main()
+    main()