Update ingest.py

ingest.py

@@ -1,39 +1,78 @@
 import os
+import logging
 from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 def create_faiss_index():
+    documents = []
+    docs_dir = "docs"
+
+    if not os.path.exists(docs_dir):
+        logger.error(f"The directory '{docs_dir}' does not exist.")
+        return
+
+    if not os.listdir(docs_dir):
+        logger.error(f"The directory '{docs_dir}' is empty.")
+        return
+
+    for root, dirs, files in os.walk(docs_dir):
+        for file in files:
+            if file.endswith(".pdf"):
+                file_path = os.path.join(root, file)
+                logger.info(f"Loading document: {file_path}")
+                try:
+                    loader = PyPDFLoader(file_path)
+                    documents.extend(loader.load())
+                    logger.info(f"Successfully loaded document: {file_path}")
+                except Exception as e:
+                    logger.error(f"Error loading {file_path}: {e}")
+
+    if not documents:
+        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
+        return
+
+    logger.info(f"Loaded {len(documents)} documents.")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
+
+    if not texts:
+        logger.error("No text chunks were created. Check the text splitting process.")
+        return
+
+    logger.info(f"Created {len(texts)} text chunks.")
+
     try:
-        # Ensure the 'docs' directory exists and contains files
-        docs_directory = 'docs'
-        if not os.path.exists(docs_directory) or not os.listdir(docs_directory):
-            raise ValueError(f"Directory '{docs_directory}' is empty or does not exist.")
-
-        # Load all documents from the 'docs' directory
-        documents = []
-        for file in os.listdir(docs_directory):
-            if file.endswith('.pdf'):
-                loader = PyPDFLoader(os.path.join(docs_directory, file))
-                documents.extend(loader.load())
-        
-        if not documents:
-            raise ValueError("No valid documents found in the 'docs' directory.")
-
-        # Create embeddings using HuggingFace's 'sentence-transformers/all-MiniLM-L6-v2' model
         embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-        
-        # Create the FAISS vector store index
-        faiss_index = FAISS.from_documents(documents, embeddings)
-
-        # Save the FAISS index locally
-        index_path = "faiss_index"
-        os.makedirs(index_path, exist_ok=True)
-        faiss_index.save_local(index_path)
-        
-        print("FAISS index created and saved successfully.")
     except Exception as e:
-        print(f"An error occurred during FAISS index creation: {e}")
+        logger.error(f"Failed to initialize embeddings: {e}")
+        return
+
+    try:
+        db = FAISS.from_documents(texts, embeddings)
+        logger.info(f"Created FAISS index with {len(texts)} vectors")
+    except Exception as e:
+        logger.error(f"Failed to create FAISS index: {e}")
+        return
+
+    index_dir = "faiss_index"
+    if not os.path.exists(index_dir):
+        os.makedirs(index_dir)
+
+    try:
+        db.save_local(index_dir)
+        index_path = os.path.join(index_dir, "index.faiss")
+        logger.info(f"FAISS index successfully saved to {index_dir}")
+        logger.info(f"Index file size after creation: {os.path.getsize(index_path)} bytes")
+        logger.info(f"Index file permissions: {oct(os.stat(index_path).st_mode)[-3:]}")
+    except Exception as e:
+        logger.error(f"Failed to save FAISS index: {e}")
 
 if __name__ == "__main__":
     create_faiss_index()