Upload 2 files

app.py

@@ -1,124 +1,124 @@
-import streamlit as st
-import os
-import faiss
-import logging
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
-from langchain_community.llms import HuggingFacePipeline
-from langchain.chains import RetrievalQA
-from ingest import create_faiss_index_from_pdfs
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-checkpoint = "LaMini-T5-738M"
-
-@st.cache_resource
-def load_llm():
-    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
-    model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
-    pipe = pipeline(
-        'text2text-generation',
-        model=model,
-        tokenizer=tokenizer,
-        max_length=256,
-        do_sample=True,
-        temperature=0.3,
-        top_p=0.95
-    )
-    return HuggingFacePipeline(pipeline=pipe)
-
-def validate_index_file(index_path):
-    try:
-        with open(index_path, 'rb') as f:
-            data = f.read(100)
-        logger.info(f"Successfully read {len(data)} bytes from the index file")
-        return True
-    except Exception as e:
-        logger.error(f"Error validating index file: {e}")
-        return False
-
-def load_faiss_index():
-    index_path = "faiss_index/index.faiss"
-    if not os.path.exists(index_path):
-        st.warning("Index file not found. Creating a new one...")
-        create_faiss_index_from_pdfs()
-
-    if not os.path.exists(index_path):
-        st.error("Failed to create the FAISS index. Please check the 'docs' directory and try again.")
-        raise RuntimeError("FAISS index creation failed.")
-
-    try:
-        index = faiss.read_index(index_path)
-        if index is None:
-            raise ValueError("Failed to read FAISS index.")
-        
-        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-        db = FAISS.load_local("faiss_index", embeddings)
-        if db.index is None or db.index_to_docstore_id is None:
-            raise ValueError("FAISS index or docstore_id mapping is None.")
-
-        return db.as_retriever()
-    except Exception as e:
-        st.error(f"Failed to load FAISS index: {e}")
-        logger.exception("Exception in load_faiss_index")
-        raise
-
-def process_answer(instruction):
-    try:
-        retriever = load_faiss_index()
-        llm = load_llm()
-        qa = RetrievalQA.from_chain_type(
-            llm=llm,
-            chain_type="stuff",
-            retriever=retriever,
-            return_source_documents=True
-        )
-        generated_text = qa.invoke(instruction)
-        answer = generated_text['result']
-        return answer, generated_text
-    except Exception as e:
-        st.error(f"An error occurred while processing the answer: {e}")
-        logger.exception("Exception in process_answer")
-        return "An error occurred while processing your request.", {}
-
-def diagnose_faiss_index():
-    index_path = "faiss_index/index.faiss"
-    if os.path.exists(index_path):
-        st.write(f"Index file size: {os.path.getsize(index_path)} bytes")
-        st.write(f"Index file permissions: {oct(os.stat(index_path).st_mode)[-3:]}")
-        st.write(f"Index file owner: {os.stat(index_path).st_uid}")
-        st.write(f"Current process user ID: {os.getuid()}")
-        validate_index_file(index_path)
-    else:
-        st.warning("Index file does not exist.")
-
-def main():
-    st.title("Search Your PDF 📚📝")
-    
-    with st.expander("About the App"):
-        st.markdown(
-            """
-            This is a Generative AI powered Question and Answering app that responds to questions about your PDF File.
-            """
-        )
-
-    diagnose_faiss_index()
-
-    question = st.text_area("Enter your Question")
-    
-    if st.button("Ask"):
-        st.info("Your Question: " + question)
-        st.info("Your Answer")
-        try:
-            answer, metadata = process_answer(question)
-            st.write(answer)
-            st.write(metadata)
-        except Exception as e:
-            st.error(f"An unexpected error occurred: {e}")
-            logger.exception("Unexpected error in main function")
-
-if __name__ == '__main__':
-    main()
+import streamlit as st
+import os
+import faiss
+import logging
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_community.llms import HuggingFacePipeline
+from langchain.chains import RetrievalQA
+from ingest import create_faiss_index_from_pdfs
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+checkpoint = "LaMini-T5-738M"
+
+@st.cache_resource
+def load_llm():
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+    model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
+    pipe = pipeline(
+        'text2text-generation',
+        model=model,
+        tokenizer=tokenizer,
+        max_length=256,
+        do_sample=True,
+        temperature=0.3,
+        top_p=0.95
+    )
+    return HuggingFacePipeline(pipeline=pipe)
+
+def validate_index_file(index_path):
+    try:
+        with open(index_path, 'rb') as f:
+            data = f.read(100)
+        logger.info(f"Successfully read {len(data)} bytes from the index file")
+        return True
+    except Exception as e:
+        logger.error(f"Error validating index file: {e}")
+        return False
+
+def load_faiss_index():
+    index_path = "faiss_index/index.faiss"
+    if not os.path.exists(index_path):
+        st.warning("Index file not found. Creating a new one...")
+        create_faiss_index_from_pdfs()
+
+    if not os.path.exists(index_path):
+        st.error("Failed to create the FAISS index. Please check the 'docs' directory and try again.")
+        raise RuntimeError("FAISS index creation failed.")
+
+    try:
+        index = faiss.read_index(index_path)
+        if index is None:
+            raise ValueError("Failed to read FAISS index.")
+        
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+        db = FAISS.load_local("faiss_index", embeddings)
+        if db.index is None or db.index_to_docstore_id is None:
+            raise ValueError("FAISS index or docstore_id mapping is None.")
+
+        return db.as_retriever()
+    except Exception as e:
+        st.error(f"Failed to load FAISS index: {e}")
+        logger.exception("Exception in load_faiss_index")
+        raise
+
+def process_answer(instruction):
+    try:
+        retriever = load_faiss_index()
+        llm = load_llm()
+        qa = RetrievalQA.from_chain_type(
+            llm=llm,
+            chain_type="stuff",
+            retriever=retriever,
+            return_source_documents=True
+        )
+        generated_text = qa.invoke(instruction)
+        answer = generated_text['result']
+        return answer, generated_text
+    except Exception as e:
+        st.error(f"An error occurred while processing the answer: {e}")
+        logger.exception("Exception in process_answer")
+        return "An error occurred while processing your request.", {}
+
+def diagnose_faiss_index():
+    index_path = "faiss_index/index.faiss"
+    if os.path.exists(index_path):
+        st.write(f"Index file size: {os.path.getsize(index_path)} bytes")
+        st.write(f"Index file permissions: {oct(os.stat(index_path).st_mode)[-3:]}")
+        st.write(f"Index file owner: {os.stat(index_path).st_uid}")
+        st.write(f"Current process user ID: {os.getuid()}")
+        validate_index_file(index_path)
+    else:
+        st.warning("Index file does not exist.")
+
+def main():
+    st.title("Search Your PDF 📚📝")
+    
+    with st.expander("About the App"):
+        st.markdown(
+            """
+            This is a Generative AI powered Question and Answering app that responds to questions about your PDF File.
+            """
+        )
+
+    diagnose_faiss_index()
+
+    question = st.text_area("Enter your Question")
+    
+    if st.button("Ask"):
+        st.info("Your Question: " + question)
+        st.info("Your Answer")
+        try:
+            answer, metadata = process_answer(question)
+            st.write(answer)
+            st.write(metadata)
+        except Exception as e:
+            st.error(f"An unexpected error occurred: {e}")
+            logger.exception("Unexpected error in main function")
+
+if __name__ == '__main__':
+    main()

ingest.py

@@ -1,99 +1,99 @@
-import os
-import logging
-from langchain_community.document_loaders import PDFMinerLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-def create_faiss_index(texts, embeddings):
-    """
-    Create a FAISS index from text chunks and embeddings.
-    :param texts: List of text chunks.
-    :param embeddings: HuggingFaceEmbeddings object.
-    :return: FAISS index object.
-    """
-    try:
-        db = FAISS.from_documents(texts, embeddings)
-        logger.info(f"Created FAISS index with {len(texts)} vectors")
-        # Check the FAISS index size
-        if len(db.index) > 0:
-            logger.info(f"FAISS index contains {len(db.index)} vectors.")
-        else:
-            logger.error("FAISS index contains 0 vectors after creation. Check the data and embeddings.")
-    except Exception as e:
-        logger.error(f"Failed to create FAISS index: {e}")
-        return None
-
-    return db
-
-def save_faiss_index(db, index_path):
-    """
-    Save the FAISS index to a specified path.
-    :param db: FAISS index object.
-    :param index_path: Path to save the index.
-    """
-    try:
-        db.save_local(index_path)
-        # Check the file size
-        index_file_path = os.path.join(index_path, "index.faiss")
-        file_size = os.path.getsize(index_file_path)
-        logger.info(f"FAISS index saved to {index_path}")
-        logger.info(f"Index file size: {file_size} bytes")
-        if file_size == 0:
-            logger.error(f"Index file '{index_file_path}' is empty.")
-    except Exception as e:
-        logger.error(f"Failed to save FAISS index to {index_path}: {e}")
-
-def create_faiss_index_from_pdfs():
-    documents = []
-    docs_dir = "docs"
-
-    if not os.path.exists(docs_dir):
-        logger.error(f"The directory '{docs_dir}' does not exist.")
-        return
-
-    for root, dirs, files in os.walk(docs_dir):
-        for file in files:
-            if file.endswith(".pdf"):
-                file_path = os.path.join(root, file)
-                logger.info(f"Loading document: {file_path}")
-                try:
-                    loader = PDFMinerLoader(file_path)
-                    documents.extend(loader.load())
-                except Exception as e:
-                    logger.error(f"Error loading {file_path}: {e}")
-
-    if not documents:
-        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
-        return
-
-    logger.info(f"Loaded {len(documents)} documents.")
-
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
-    texts = text_splitter.split_documents(documents)
-
-    if not texts:
-        logger.error("No text chunks were created. Check the text splitting process.")
-        return
-
-    logger.info(f"Created {len(texts)} text chunks.")
-
-    try:
-        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-    except Exception as e:
-        logger.error(f"Failed to initialize embeddings: {e}")
-        return
-
-    db = create_faiss_index(texts, embeddings)
-    if db:
-        index_dir = "faiss_index"
-        if not os.path.exists(index_dir):
-            os.makedirs(index_dir)
-        save_faiss_index(db, index_dir)
-
-if __name__ == "__main__":
-    create_faiss_index_from_pdfs()
+import os
+import logging
+from langchain_community.document_loaders import PDFMinerLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def create_faiss_index(texts, embeddings):
+    """
+    Create a FAISS index from text chunks and embeddings.
+    :param texts: List of text chunks.
+    :param embeddings: HuggingFaceEmbeddings object.
+    :return: FAISS index object.
+    """
+    try:
+        db = FAISS.from_documents(texts, embeddings)
+        logger.info(f"Created FAISS index with {len(texts)} vectors")
+        # Check the FAISS index size
+        if len(db.index) > 0:
+            logger.info(f"FAISS index contains {len(db.index)} vectors.")
+        else:
+            logger.error("FAISS index contains 0 vectors after creation. Check the data and embeddings.")
+    except Exception as e:
+        logger.error(f"Failed to create FAISS index: {e}")
+        return None
+
+    return db
+
+def save_faiss_index(db, index_path):
+    """
+    Save the FAISS index to a specified path.
+    :param db: FAISS index object.
+    :param index_path: Path to save the index.
+    """
+    try:
+        db.save_local(index_path)
+        # Check the file size
+        index_file_path = os.path.join(index_path, "index.faiss")
+        file_size = os.path.getsize(index_file_path)
+        logger.info(f"FAISS index saved to {index_path}")
+        logger.info(f"Index file size: {file_size} bytes")
+        if file_size == 0:
+            logger.error(f"Index file '{index_file_path}' is empty.")
+    except Exception as e:
+        logger.error(f"Failed to save FAISS index to {index_path}: {e}")
+
+def create_faiss_index_from_pdfs():
+    documents = []
+    docs_dir = "docs"
+
+    if not os.path.exists(docs_dir):
+        logger.error(f"The directory '{docs_dir}' does not exist.")
+        return
+
+    for root, dirs, files in os.walk(docs_dir):
+        for file in files:
+            if file.endswith(".pdf"):
+                file_path = os.path.join(root, file)
+                logger.info(f"Loading document: {file_path}")
+                try:
+                    loader = PDFMinerLoader(file_path)
+                    documents.extend(loader.load())
+                except Exception as e:
+                    logger.error(f"Error loading {file_path}: {e}")
+
+    if not documents:
+        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
+        return
+
+    logger.info(f"Loaded {len(documents)} documents.")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
+
+    if not texts:
+        logger.error("No text chunks were created. Check the text splitting process.")
+        return
+
+    logger.info(f"Created {len(texts)} text chunks.")
+
+    try:
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    except Exception as e:
+        logger.error(f"Failed to initialize embeddings: {e}")
+        return
+
+    db = create_faiss_index(texts, embeddings)
+    if db:
+        index_dir = "faiss_index"
+        if not os.path.exists(index_dir):
+            os.makedirs(index_dir)
+        save_faiss_index(db, index_dir)
+
+if __name__ == "__main__":
+    create_faiss_index_from_pdfs()