Update ingest.py

ingest.py

@@ -1,15 +1,20 @@
 import os
 import logging
-from langchain.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
 
-# Setup logging
+# Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def load_documents(docs_dir):
+    """
+    Load documents from a directory.
+    :param docs_dir: Directory containing PDF documents.
+    :return: List of loaded documents.
+    """
     documents = []
     for root, dirs, files in os.walk(docs_dir):
         for file in files:
@@ -29,6 +34,11 @@ def load_documents(docs_dir):
     return documents
 
 def split_text(documents):
+    """
+    Split documents into text chunks.
+    :param documents: List of documents to be split.
+    :return: List of text chunks.
+    """
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
     texts = text_splitter.split_documents(documents)
 
@@ -43,6 +53,10 @@ def split_text(documents):
     return texts
 
 def create_embeddings():
+    """
+    Create embeddings using a HuggingFace model.
+    :return: HuggingFaceEmbeddings object.
+    """
     model_name = "sentence-transformers/all-MiniLM-L6-v2"
     embeddings = HuggingFaceEmbeddings(model_name=model_name)
     
@@ -56,6 +70,12 @@ def create_embeddings():
     return embeddings
 
 def create_faiss_index(texts, embeddings):
+    """
+    Create a FAISS index from text chunks and embeddings.
+    :param texts: List of text chunks.
+    :param embeddings: HuggingFaceEmbeddings object.
+    :return: FAISS index object.
+    """
     try:
         db = FAISS.from_documents(texts, embeddings)
         logger.info(f"Created FAISS index with {len(texts)} vectors")
@@ -71,6 +91,11 @@ def create_faiss_index(texts, embeddings):
     return db
 
 def save_faiss_index(db, index_path):
+    """
+    Save the FAISS index to a specified path.
+    :param db: FAISS index object.
+    :param index_path: Path to save the index.
+    """
     try:
         db.save_local(index_path)
         logger.info(f"FAISS index saved to {index_path}")
@@ -110,5 +135,5 @@ def main():
     # Save FAISS index
     save_faiss_index(db, index_path)
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     main()