Update app.py

app.py

@@ -7,6 +7,11 @@ from langchain.embeddings import HuggingFaceEmbeddings
 from openai import OpenAI
 from langchain_openai import ChatOpenAI
 from typing import List, Dict
+import fitz  # PyMuPDF
+from langchain.schema import Document
+from langchain_experimental.text_splitter import SemanticChunker # module for chunking text
+import os
+
 # Load environment variables
 OPENAI_API_KEY = os.getenv("OPENAI_API")
 TOKEN=os.getenv('HF_TOKEN')
@@ -20,7 +25,45 @@ class PDFChatbot:
 
     def get_relevant_context(self, user_question: str) -> List[str]:
         """Split text into smaller chunks for better processing."""
-        db = FAISS.load_local('mbaldb', HuggingFaceEmbeddings(model_name='bkai-foundation-models/vietnamese-bi-encoder'), allow_dangerous_deserialization = True )
+        # db = FAISS.load_local('mbaldb', HuggingFaceEmbeddings(model_name='bkai-foundation-models/vietnamese-bi-encoder'), allow_dangerous_deserialization = True )
+
+
+        pdf_directory = "data"
+        
+        # Duyệt qua các file trong thư mục và đọc từng file PDF
+        pdf_texts = []
+        for filename in os.listdir(pdf_directory):
+            if filename.endswith(".pdf"):
+                file_path = os.path.join(pdf_directory, filename)
+        
+                # Mở file PDF
+                doc = fitz.open(file_path)
+        
+                # Trích xuất toàn bộ văn bản từ từng trang
+                full_text = ""
+                for page_num in range(doc.page_count):
+                    page = doc.load_page(page_num)
+                    full_text += page.get_text("text", flags=11)
+        
+                pdf_texts.append({"file": filename, "text": full_text})
+        
+        documents = [
+            Document(page_content=doc['text'], metadata={'file': doc['file']})
+            for doc in pdf_texts  # Assuming pdf_texts is a list of dictionaries like {'file': filename, 'text': full_text}
+        ]
+        
+        semantic_splitter = SemanticChunker(
+            embeddings= HuggingFaceEmbeddings(model_name='bkai-foundation-models/vietnamese-bi-encoder'),
+            buffer_size=1, # total sentence collected before perform text split
+            breakpoint_threshold_type='percentile', # set splitting style: 'percentage' of similarity
+            breakpoint_threshold_amount=95, # split text if similarity score > 95%
+            min_chunk_size=500,
+            add_start_index=True, # assign index for chunk
+        )
+        
+        docs = semantic_splitter.split_documents(documents)
+        db = FAISS.from_documents(docs, HuggingFaceEmbeddings(model_name='bkai-foundation-models/vietnamese-bi-encoder'))
+
         relevant_chunks = db.similarity_search(user_question, k=3)
         relevant_chunks = [chunk.page_content for chunk in relevant_chunks]
         return "\n\n".join(relevant_chunks)