Update ingest.py

ingest.py

@@ -2,26 +2,15 @@ import os
 import logging
 from langchain.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
 
-# Set up logging
-logging.basicConfig(level=logging.DEBUG)
+# Setup logging
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def create_faiss_index():
+def load_documents(docs_dir):
     documents = []
-    docs_dir = "docs"
-
-    if not os.path.exists(docs_dir):
-        logger.error(f"The directory '{docs_dir}' does not exist.")
-        return
-
-    if not os.listdir(docs_dir):
-        logger.error(f"The directory '{docs_dir}' is empty.")
-        return
-
-    # Load documents
     for root, dirs, files in os.walk(docs_dir):
         for file in files:
             if file.endswith(".pdf"):
@@ -34,71 +23,92 @@ def create_faiss_index():
                         documents.extend(loaded_docs)
                         logger.info(f"Loaded {len(loaded_docs)} pages from {file_path}.")
                     else:
-                        logger.warning(f"No content extracted from {file_path}.")
+                        logger.warning(f"No content extracted from {file_path}. Possibly encrypted or empty.")
                 except Exception as e:
                     logger.error(f"Error loading {file_path}: {e}")
+    return documents
 
-    if not documents:
-        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
-        return
-
-    logger.info(f"Total loaded documents: {len(documents)}")
-
-    # Split text into chunks
+def split_text(documents):
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
     texts = text_splitter.split_documents(documents)
 
     if not texts:
         logger.error("No text chunks were created. Check the text splitting process.")
-        return
+        return None
 
     logger.info(f"Created {len(texts)} text chunks.")
-
-    # Check a sample of text chunks
-    for i, text in enumerate(texts[:5]):
-        logger.debug(f"Sample chunk {i}: {text[:100]}...")
-
-    # Create embeddings
-    try:
-        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-    except Exception as e:
-        logger.error(f"Failed to initialize embeddings: {e}")
-        return
-
-    # Verify embeddings are valid by generating one
+    for i, text in enumerate(texts[:5]):  # Sample first 5 chunks
+        logger.debug(f"Sample chunk {i}: {text[:100]}...")  # Print first 100 characters
+    
+    return texts
+
+def create_embeddings():
+    model_name = "sentence-transformers/all-MiniLM-L6-v2"
+    embeddings = HuggingFaceEmbeddings(model_name=model_name)
+    
     try:
         sample_embedding = embeddings.embed_query("sample text")
         logger.debug(f"Sample embedding: {sample_embedding[:5]}... (truncated for brevity)")
     except Exception as e:
         logger.error(f"Error generating sample embedding: {e}")
-        return
+        return None
 
-    # Create FAISS index
+    return embeddings
+
+def create_faiss_index(texts, embeddings):
     try:
         db = FAISS.from_documents(texts, embeddings)
         logger.info(f"Created FAISS index with {len(texts)} vectors")
+        # Directly check the FAISS index size
+        if len(db.index) > 0:
+            logger.info(f"FAISS index contains {len(db.index)} vectors.")
+        else:
+            logger.error("FAISS index contains 0 vectors after creation. Check the data and embeddings.")
     except Exception as e:
         logger.error(f"Failed to create FAISS index: {e}")
-        return
+        return None
 
-    # Save FAISS index locally
-    index_dir = "faiss_index"
-    if not os.path.exists(index_dir):
-        os.makedirs(index_dir)
+    return db
 
+def save_faiss_index(db, index_path):
     try:
-        db.save_local(index_dir)
-        index_path = os.path.join(index_dir, "index.faiss")
-        if os.path.exists(index_path):
-            logger.info(f"FAISS index file exists. Size: {os.path.getsize(index_path)} bytes.")
-            if os.path.getsize(index_path) > 0:
-                logger.info(f"FAISS index successfully saved to {index_path}")
-            else:
-                logger.error(f"FAISS index file '{index_path}' is empty.")
-        else:
-            logger.error(f"FAISS index file '{index_path}' does not exist after save attempt.")
+        db.save_local(index_path)
+        logger.info(f"FAISS index saved to {index_path}")
     except Exception as e:
-        logger.error(f"Failed to save FAISS index: {e}")
+        logger.error(f"Failed to save FAISS index to {index_path}: {e}")
+
+def main():
+    docs_dir = "docs"  # Adjust to your document directory
+    index_path = "faiss_index"
+
+    logger.info("Starting document processing...")
+
+    # Load documents
+    documents = load_documents(docs_dir)
+    if not documents:
+        logger.error("No documents were loaded. Exiting.")
+        return
+
+    # Split text into chunks
+    texts = split_text(documents)
+    if texts is None:
+        logger.error("Text splitting failed. Exiting.")
+        return
+
+    # Create embeddings
+    embeddings = create_embeddings()
+    if embeddings is None:
+        logger.error("Embeddings creation failed. Exiting.")
+        return
+
+    # Create FAISS index
+    db = create_faiss_index(texts, embeddings)
+    if db is None:
+        logger.error("FAISS index creation failed. Exiting.")
+        return
+
+    # Save FAISS index
+    save_faiss_index(db, index_path)
 
 if __name__ == "__main__":
-    create_faiss_index()
+    main()