Spaces:
Build error
Build error
File size: 2,729 Bytes
2354330 d343dde 2354330 d343dde 2354330 d343dde 2354330 d343dde 2354330 d343dde 2354330 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
import os
import logging
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain_community.vectorstores import FAISS
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def create_faiss_index():
documents = []
docs_dir = "docs"
if not os.path.exists(docs_dir):
logger.error(f"The directory '{docs_dir}' does not exist.")
return
if not os.listdir(docs_dir):
logger.error(f"The directory '{docs_dir}' is empty.")
return
for root, dirs, files in os.walk(docs_dir):
for file in files:
if file.endswith(".pdf"):
file_path = os.path.join(root, file)
logger.info(f"Loading document: {file_path}")
try:
loader = PyPDFLoader(file_path)
documents.extend(loader.load())
logger.info(f"Successfully loaded document: {file_path}")
except Exception as e:
logger.error(f"Error loading {file_path}: {e}")
if not documents:
logger.error("No documents were loaded. Check the 'docs' directory and file paths.")
return
logger.info(f"Loaded {len(documents)} documents.")
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
texts = text_splitter.split_documents(documents)
if not texts:
logger.error("No text chunks were created. Check the text splitting process.")
return
logger.info(f"Created {len(texts)} text chunks.")
try:
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
except Exception as e:
logger.error(f"Failed to initialize embeddings: {e}")
return
try:
db = FAISS.from_documents(texts, embeddings)
logger.info(f"Created FAISS index with {len(texts)} vectors")
except Exception as e:
logger.error(f"Failed to create FAISS index: {e}")
return
index_dir = "faiss_index"
if not os.path.exists(index_dir):
os.makedirs(index_dir)
try:
db.save_local(index_dir)
index_path = os.path.join(index_dir, "index.faiss")
logger.info(f"FAISS index successfully saved to {index_dir}")
logger.info(f"Index file size after creation: {os.path.getsize(index_path)} bytes")
logger.info(f"Index file permissions: {oct(os.stat(index_path).st_mode)[-3:]}")
except Exception as e:
logger.error(f"Failed to save FAISS index: {e}")
if __name__ == "__main__":
create_faiss_index()
|