Update file_loader.py

file_loader.py

@@ -60,7 +60,7 @@ def get_vectorstore():
     # print('Vectorstore ready!')
     # return vectorstore
 
-    folder_path = "syllabus_nct_word_format/" #'/content/chatbot4nct_test2/syllabus_nct_word_format'
+    folder_path = "syllabus_nct_pdf_format/" #'/content/chatbot4nct_test2/syllabus_nct_word_format'
     docx_files = list_docx_files(folder_path)
     all_splits = []  # Khởi tạo danh sách lưu kết quả
     for i, file_path in enumerate(tqdm(docx_files, desc="Đang xử lý", unit="file")):
@@ -69,7 +69,15 @@ def get_vectorstore():
         splits = get_splits(file_path, output_json_path)
         splits_with_metadata = update_documents_metadata(splits, metadata)
         all_splits += splits_with_metadata
-        if i == 1: break
+        # if i == 1: break
+
+    FAQ_path = "syllabus_nct_word_format/FAQ.json"
+    FAQ_splits = get_json_splits_only(FAQ_path)
+    all_splits += FAQ_splits
+
+    FAQ_path = "syllabus_nct_word_format/FAQ2.json"
+    FAQ_splits = get_json_splits_only(FAQ_path)
+    all_splits += FAQ_splits
     
     print('Get embedding model /paraphrase-multilingual-MiniLM-L12-v2')
     embedding = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")   #"VoVanPhuc/sup-SimCSE-VietNamese-phobert-base")