Update ingest.py

ingest.py

@@ -1,22 +1,25 @@
 import os
 import logging
 import faiss
-import numpy as np
 from langchain_community.document_loaders import PDFMinerLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
 
+# Configure logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 def create_faiss_index():
     documents = []
-    docs_dir = "docs"
+    docs_dir = "docs"  # Directory where PDF files are stored
 
+    # Check if the 'docs' directory exists
     if not os.path.exists(docs_dir):
         logger.error(f"The directory '{docs_dir}' does not exist.")
         return
 
+    # Walk through the 'docs' directory and load PDF files
     for root, dirs, files in os.walk(docs_dir):
         for file in files:
             if file.endswith(".pdf"):
@@ -33,38 +36,39 @@ def create_faiss_index():
                 except Exception as e:
                     logger.error(f"Error loading {file_path}: {e}")
 
+    # Check if any documents were loaded
     if not documents:
         logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
         return
 
     logger.info(f"Loaded {len(documents)} documents.")
 
+    # Split documents into text chunks
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
     texts = text_splitter.split_documents(documents)
     logger.info(f"Created {len(texts)} text chunks.")
+
+    # Check if text chunks were created
     if not texts:
         logger.error("No text chunks created. Check the text splitting process.")
         return
 
     try:
+        # Initialize embeddings using HuggingFace models
         embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
         logger.info("Embeddings initialized successfully.")
     except Exception as e:
         logger.error(f"Failed to initialize embeddings: {e}")
         return
 
-    embedding_vectors = np.array([embeddings.embed(text) for text in texts])
-    dimension = embedding_vectors.shape[1]
-    
     try:
-        faiss_index = faiss.IndexFlatL2(dimension)
-        faiss_index.add(embedding_vectors)
-        os.makedirs("faiss_index", exist_ok=True)  # Create directory if it doesn't exist
-        faiss.write_index(faiss_index, "faiss_index/index.faiss")
+        # Create a FAISS index and save it
+        index = faiss.IndexFlatL2(embeddings.embedding_size)
+        vector_store = FAISS.from_documents(texts, embeddings, index)
+        vector_store.save_local("faiss_index")
         logger.info(f"Created FAISS index with {len(texts)} vectors.")
     except Exception as e:
         logger.error(f"Failed to create FAISS index: {e}")
-        return
 
 if __name__ == "__main__":
     create_faiss_index()