Update ingest.py

ingest.py

@@ -13,53 +13,54 @@ def create_faiss_index():
     docs_dir = "docs"
 
     if not os.path.exists(docs_dir):
-        print(f"The directory '{docs_dir}' does not exist.")
+        logger.error(f"The directory '{docs_dir}' does not exist.")
         return
 
     for root, dirs, files in os.walk(docs_dir):
         for file in files:
             if file.endswith(".pdf"):
                 file_path = os.path.join(root, file)
-                print(f"Loading document: {file_path}")
+                logger.info(f"Loading document: {file_path}")
                 try:
                     loader = PDFMinerLoader(file_path)
                     loaded_docs = loader.load()
                     if loaded_docs:
-                        print(f"Loaded {len(loaded_docs)} documents from {file_path}")
+                        logger.info(f"Loaded {len(loaded_docs)} documents from {file_path}")
+                        documents.extend(loaded_docs)
                     else:
-                        print(f"No documents loaded from {file_path}")
-                    documents.extend(loaded_docs)
+                        logger.warning(f"No documents loaded from {file_path}")
                 except Exception as e:
-                    print(f"Error loading {file_path}: {e}")
+                    logger.error(f"Error loading {file_path}: {e}")
 
     if not documents:
-        print("No documents were loaded. Check the 'docs' directory and file paths.")
+        logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
         return
 
-    print(f"Loaded {len(documents)} documents.")
+    logger.info(f"Loaded {len(documents)} documents.")
 
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
     texts = text_splitter.split_documents(documents)
-    print(f"Created {len(texts)} text chunks.")
+    logger.info(f"Created {len(texts)} text chunks.")
     if not texts:
-        print("No text chunks created. Check the text splitting process.")
+        logger.error("No text chunks created. Check the text splitting process.")
         return
 
     try:
         embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
-        print("Embeddings initialized successfully.")
+        logger.info("Embeddings initialized successfully.")
     except Exception as e:
-        print(f"Failed to initialize embeddings: {e}")
+        logger.error(f"Failed to initialize embeddings: {e}")
         return
 
     try:
         db = FAISS.from_documents(texts, embeddings)
         if db.index.ntotal > 0:
-            print(f"Created FAISS index with {db.index.ntotal} vectors.")
+            logger.info(f"Created FAISS index with {db.index.ntotal} vectors.")
         else:
-            print("FAISS index contains 0 vectors.")
+            logger.error("FAISS index contains 0 vectors.")
+            return
     except Exception as e:
-        print(f"Failed to create FAISS index: {e}")
+        logger.error(f"Failed to create FAISS index: {e}")
         return
 
     index_dir = "faiss_index"
@@ -70,14 +71,14 @@ def create_faiss_index():
         db.save_local(index_dir)
         index_file_path = os.path.join(index_dir, "index.faiss")
         file_size = os.path.getsize(index_file_path)
-        print(f"FAISS index saved to {index_dir}")
-        print(f"Index file size: {file_size} bytes")
+        logger.info(f"FAISS index saved to {index_dir}")
+        logger.info(f"Index file size: {file_size} bytes")
         if file_size == 0:
-            print(f"Index file '{index_file_path}' is empty.")
+            logger.error(f"Index file '{index_file_path}' is empty.")
         else:
-            print(f"Index file '{index_file_path}' created successfully.")
+            logger.info(f"Index file '{index_file_path}' created successfully.")
     except Exception as e:
-        print(f"Failed to save FAISS index: {e}")
+        logger.error(f"Failed to save FAISS index: {e}")
 
 if __name__ == "__main__":
     create_faiss_index()