Update ingest.py

ingest.py

@@ -1,6 +1,6 @@
 import os
 import logging
-from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import PDFMinerLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
@@ -9,66 +9,6 @@ from langchain_community.vectorstores import FAISS
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def load_documents(docs_dir):
-    """
-    Load documents from a directory.
-    :param docs_dir: Directory containing PDF documents.
-    :return: List of loaded documents.
-    """
-    documents = []
-    for root, dirs, files in os.walk(docs_dir):
-        for file in files:
-            if file.endswith(".pdf"):
-                file_path = os.path.join(root, file)
-                logger.info(f"Loading document: {file_path}")
-                try:
-                    loader = PyPDFLoader(file_path)
-                    loaded_docs = loader.load()
-                    if loaded_docs:
-                        documents.extend(loaded_docs)
-                        logger.info(f"Loaded {len(loaded_docs)} pages from {file_path}.")
-                    else:
-                        logger.warning(f"No content extracted from {file_path}. Possibly encrypted or empty.")
-                except Exception as e:
-                    logger.error(f"Error loading {file_path}: {e}")
-    return documents
-
-def split_text(documents):
-    """
-    Split documents into text chunks.
-    :param documents: List of documents to be split.
-    :return: List of text chunks.
-    """
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
-    texts = text_splitter.split_documents(documents)
-
-    if not texts:
-        logger.error("No text chunks were created. Check the text splitting process.")
-        return None
-
-    logger.info(f"Created {len(texts)} text chunks.")
-    for i, text in enumerate(texts[:5]):  # Sample first 5 chunks
-        logger.debug(f"Sample chunk {i}: {text[:100]}...")  # Print first 100 characters
-    
-    return texts
-
-def create_embeddings():
-    """
-    Create embeddings using a HuggingFace model.
-    :return: HuggingFaceEmbeddings object.
-    """
-    model_name = "sentence-transformers/all-MiniLM-L6-v2"
-    embeddings = HuggingFaceEmbeddings(model_name=model_name)
-    
-    try:
-        sample_embedding = embeddings.embed_query("sample text")
-        logger.debug(f"Sample embedding: {sample_embedding[:5]}... (truncated for brevity)")
-    except Exception as e:
-        logger.error(f"Error generating sample embedding: {e}")
-        return None
-
-    return embeddings
-
 def create_faiss_index(texts, embeddings):
     """
     Create a FAISS index from text chunks and embeddings.
@@ -79,7 +19,7 @@ def create_faiss_index(texts, embeddings):
     try:
         db = FAISS.from_documents(texts, embeddings)
         logger.info(f"Created FAISS index with {len(texts)} vectors")
-        # Directly check the FAISS index size
+        # Check the FAISS index size
         if len(db.index) > 0:
             logger.info(f"FAISS index contains {len(db.index)} vectors.")
         else:
@@ -98,42 +38,62 @@ def save_faiss_index(db, index_path):
     """
     try:
         db.save_local(index_path)
+        # Check the file size
+        index_file_path = os.path.join(index_path, "index.faiss")
+        file_size = os.path.getsize(index_file_path)
         logger.info(f"FAISS index saved to {index_path}")
+        logger.info(f"Index file size: {file_size} bytes")
+        if file_size == 0:
+            logger.error(f"Index file '{index_file_path}' is empty.")
     except Exception as e:
         logger.error(f"Failed to save FAISS index to {index_path}: {e}")
 
-def main():
-    docs_dir = "docs"  # Adjust to your document directory
-    index_path = "faiss_index"
+def create_faiss_index_from_pdfs():
+    documents = []
+    docs_dir = "docs"
 
-    logger.info("Starting document processing...")
+    if not os.path.exists(docs_dir):
+        logger.error(f"The directory '{docs_dir}' does not exist.")
+        return
+
+    for root, dirs, files in os.walk(docs_dir):
+        for file in files:
+            if file.endswith(".pdf"):
+                file_path = os.path.join(root, file)
+                logger.info(f"Loading document: {file_path}")
+                try:
+                    loader = PDFMinerLoader(file_path)
+                    documents.extend(loader.load())
+                except Exception as e:
+                    logger.error(f"Error loading {file_path}: {e}")
 
-    # Load documents
-    documents = load_documents(docs_dir)
     if not documents:
-        logger.error("No documents were loaded. Exiting.")
+        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
         return
 
-    # Split text into chunks
-    texts = split_text(documents)
-    if texts is None:
-        logger.error("Text splitting failed. Exiting.")
-        return
+    logger.info(f"Loaded {len(documents)} documents.")
 
-    # Create embeddings
-    embeddings = create_embeddings()
-    if embeddings is None:
-        logger.error("Embeddings creation failed. Exiting.")
-        return
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
 
-    # Create FAISS index
-    db = create_faiss_index(texts, embeddings)
-    if db is None:
-        logger.error("FAISS index creation failed. Exiting.")
+    if not texts:
+        logger.error("No text chunks were created. Check the text splitting process.")
         return
 
-    # Save FAISS index
-    save_faiss_index(db, index_path)
+    logger.info(f"Created {len(texts)} text chunks.")
+
+    try:
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    except Exception as e:
+        logger.error(f"Failed to initialize embeddings: {e}")
+        return
 
-if __name__ == '__main__':
-    main()
+    db = create_faiss_index(texts, embeddings)
+    if db:
+        index_dir = "faiss_index"
+        if not os.path.exists(index_dir):
+            os.makedirs(index_dir)
+        save_faiss_index(db, index_dir)
+
+if __name__ == "__main__":
+    create_faiss_index_from_pdfs()