Spaces:
Runtime error
Runtime error
| import os | |
| from tqdm import tqdm | |
| from langchain_community.vectorstores import FAISS | |
| from langchain_huggingface import HuggingFaceEmbeddings | |
| # from langchain_google_genai import GoogleGenerativeAIEmbeddings | |
| # Import từ helpers | |
| from helpers import ( | |
| list_docx_files, # Lấy danh sách file .docx | |
| get_splits, # Xử lý file docx thành splits | |
| get_json_splits_only, # Xử lý file JSON (FAQ) | |
| get_web_documents, # Xử lý dữ liệu từ web | |
| update_documents_metadata, | |
| define_metadata | |
| ) | |
| import json | |
| def get_vectorstore(): | |
| ### Xử lý tất cả các tài liệu và nhét vào database | |
| folder_path = "syllabus_nct_word_format/" | |
| #'syllabus_nct_word_format/Trường Công nghệ/Chương trình An toàn thông tin' | |
| #"syllabus_nct_word_format/" | |
| #'syllabus_nct_word_format/Trường Công nghệ/debug' | |
| # 'syllabus_nct_word_format/Trường Công nghệ/Chương trình An toàn thông tin' | |
| # | |
| docx_files = list_docx_files(folder_path) | |
| all_splits = [] # Khởi tạo danh sách lưu kết quả | |
| print("Feeding relevent websites' contents") | |
| # | |
| # with open('syllabus_nct_word_format/urls.txt', 'r') as f: | |
| # base_urls = [line.strip() for line in f] | |
| # urls_list | |
| # base_urls =['https://fda.neu.edu.vn/hoi-nghi-khoa-hoc-cong-nghe-dai-hoc-kinh-te-quoc-dan-nam-2025/'] | |
| # ['https://nct.neu.edu.vn/', 'https://fsf.neu.edu.vn/', 'https://mfe.neu.edu.vn/', 'https://mis.neu.edu.vn/', 'https://fda.neu.edu.vn/', 'https://khoathongke.neu.edu.vn/', 'https://fit.neu.edu.vn/'] | |
| print('Feeding .docx files') | |
| error_files = [] # List to store files that caused errors | |
| for i, file_path in enumerate(tqdm(docx_files, desc="Đang xử lý", unit="file")): | |
| # print(f'Reading {file_path}') | |
| output_json_path = f"output_{i}.json" | |
| try: | |
| metadata = define_metadata(file_path) | |
| splits = get_splits(file_path, output_json_path) | |
| splits_with_metadata = update_documents_metadata(splits, metadata) | |
| all_splits += splits_with_metadata | |
| except Exception as e: | |
| print(f"Error processing {file_path}: {e}") | |
| error_files.append(file_path) # Store the problematic file | |
| # Print all files that caused errors | |
| if error_files: | |
| print("\nFiles that caused errors:") | |
| for file in error_files: | |
| print(file) | |
| else: | |
| print("\nAll files processed successfully!") | |
| print('Feeding .json files') | |
| # Xử lý FAQ | |
| # FAQ_path = "syllabus_nct_word_format/FAQ.json" | |
| # FAQ_splits = get_json_splits_only(FAQ_path) | |
| # all_splits += FAQ_splits | |
| # FAQ_path = "syllabus_nct_word_format/FAQ2.json" | |
| # FAQ_splits = get_json_splits_only(FAQ_path) | |
| # all_splits += FAQ_splits | |
| # Lưu vào vectorstore với nhúng từ Google GenAI | |
| # embedding = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004") | |
| print('Get embedding model /paraphrase-multilingual-MiniLM-L12-v2') | |
| embedding = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2") #"VoVanPhuc/sup-SimCSE-VietNamese-phobert-base") | |
| print('Set vectorstore FAISS') | |
| vectorstore = FAISS.from_documents(documents=all_splits, embedding=embedding) | |
| return vectorstore | |