Update ingest.py

ingest.py

@@ -1,99 +1,81 @@
-import os
-import logging
-from langchain_community.document_loaders import PDFMinerLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-def create_faiss_index(texts, embeddings):
-    """
-    Create a FAISS index from text chunks and embeddings.
-    :param texts: List of text chunks.
-    :param embeddings: HuggingFaceEmbeddings object.
-    :return: FAISS index object.
-    """
-    try:
-        db = FAISS.from_documents(texts, embeddings)
-        logger.info(f"Created FAISS index with {len(texts)} vectors")
-        # Check the FAISS index size
-        if len(db.index) > 0:
-            logger.info(f"FAISS index contains {len(db.index)} vectors.")
-        else:
-            logger.error("FAISS index contains 0 vectors after creation. Check the data and embeddings.")
-    except Exception as e:
-        logger.error(f"Failed to create FAISS index: {e}")
-        return None
-
-    return db
-
-def save_faiss_index(db, index_path):
-    """
-    Save the FAISS index to a specified path.
-    :param db: FAISS index object.
-    :param index_path: Path to save the index.
-    """
-    try:
-        db.save_local(index_path)
-        # Check the file size
-        index_file_path = os.path.join(index_path, "index.faiss")
-        file_size = os.path.getsize(index_file_path)
-        logger.info(f"FAISS index saved to {index_path}")
-        logger.info(f"Index file size: {file_size} bytes")
-        if file_size == 0:
-            logger.error(f"Index file '{index_file_path}' is empty.")
-    except Exception as e:
-        logger.error(f"Failed to save FAISS index to {index_path}: {e}")
-
-def create_faiss_index_from_pdfs():
-    documents = []
-    docs_dir = "docs"
-
-    if not os.path.exists(docs_dir):
-        logger.error(f"The directory '{docs_dir}' does not exist.")
-        return
-
-    for root, dirs, files in os.walk(docs_dir):
-        for file in files:
-            if file.endswith(".pdf"):
-                file_path = os.path.join(root, file)
-                logger.info(f"Loading document: {file_path}")
-                try:
-                    loader = PDFMinerLoader(file_path)
-                    documents.extend(loader.load())
-                except Exception as e:
-                    logger.error(f"Error loading {file_path}: {e}")
-
-    if not documents:
-        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
-        return
-
-    logger.info(f"Loaded {len(documents)} documents.")
-
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
-    texts = text_splitter.split_documents(documents)
-
-    if not texts:
-        logger.error("No text chunks were created. Check the text splitting process.")
-        return
-
-    logger.info(f"Created {len(texts)} text chunks.")
-
-    try:
-        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-    except Exception as e:
-        logger.error(f"Failed to initialize embeddings: {e}")
-        return
-
-    db = create_faiss_index(texts, embeddings)
-    if db:
-        index_dir = "faiss_index"
-        if not os.path.exists(index_dir):
-            os.makedirs(index_dir)
-        save_faiss_index(db, index_dir)
-
-if __name__ == "__main__":
-    create_faiss_index_from_pdfs()
+import os
+import logging
+from langchain_community.document_loaders import PDFMinerLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def create_faiss_index():
+    documents = []
+    docs_dir = "docs"
+
+    if not os.path.exists(docs_dir):
+        print(f"The directory '{docs_dir}' does not exist.")
+        return
+
+    for root, dirs, files in os.walk(docs_dir):
+        for file in files:
+            if file.endswith(".pdf"):
+                file_path = os.path.join(root, file)
+                print(f"Loading document: {file_path}")
+                try:
+                    loader = PDFMinerLoader(file_path)
+                    loaded_docs = loader.load()
+                    if loaded_docs:
+                        print(f"Loaded {len(loaded_docs)} documents from {file_path}")
+                    else:
+                        print(f"No documents loaded from {file_path}")
+                    documents.extend(loaded_docs)
+                except Exception as e:
+                    print(f"Error loading {file_path}: {e}")
+
+    if not documents:
+        print("No documents were loaded. Check the 'docs' directory and file paths.")
+        return
+
+    print(f"Loaded {len(documents)} documents.")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
+    print(f"Created {len(texts)} text chunks.")
+    if not texts:
+        print("No text chunks created. Check the text splitting process.")
+        return
+
+    try:
+        embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+        print("Embeddings initialized successfully.")
+    except Exception as e:
+        print(f"Failed to initialize embeddings: {e}")
+        return
+
+    try:
+        db = FAISS.from_documents(texts, embeddings)
+        if db.index.ntotal > 0:
+            print(f"Created FAISS index with {db.index.ntotal} vectors.")
+        else:
+            print("FAISS index contains 0 vectors.")
+    except Exception as e:
+        print(f"Failed to create FAISS index: {e}")
+        return
+
+    index_dir = "faiss_index"
+    if not os.path.exists(index_dir):
+        os.makedirs(index_dir)
+
+    try:
+        db.save_local(index_dir)
+        index_file_path = os.path.join(index_dir, "index.faiss")
+        file_size = os.path.getsize(index_file_path)
+        print(f"FAISS index saved to {index_dir}")
+        print(f"Index file size: {file_size} bytes")
+        if file_size == 0:
+            print(f"Index file '{index_file_path}' is empty.")
+    except Exception as e:
+        print(f"Failed to save FAISS index: {e}")
+
+if __name__ == "__main__":
+    create_faiss_index()