import os
from tqdm import tqdm
from langchain_community.vectorstores import FAISS
from langchain_huggingface import HuggingFaceEmbeddings
# from langchain_google_genai import GoogleGenerativeAIEmbeddings

# Import từ helpers
from helpers import (
    list_docx_files,  # Lấy danh sách file .docx
    get_splits,  # Xử lý file docx thành splits
    get_json_splits_only,  # Xử lý file JSON (FAQ)  
    get_web_documents,  # Xử lý dữ liệu từ web
    update_documents_metadata,
    define_metadata
)

import json

def get_vectorstore():
    ### Xử lý tất cả các tài liệu và nhét vào database
    folder_path = "syllabus_nct_word_format/"
    #'syllabus_nct_word_format/Trường Công nghệ/Chương trình An toàn thông tin'
    #"syllabus_nct_word_format/"
    #'syllabus_nct_word_format/Trường Công nghệ/debug'
    # 'syllabus_nct_word_format/Trường Công nghệ/Chương trình An toàn thông tin'
    #
    docx_files = list_docx_files(folder_path)
    
    all_splits = []  # Khởi tạo danh sách lưu kết quả
    print("Feeding relevent websites' contents")
    # 
    # with open('syllabus_nct_word_format/urls.txt', 'r') as f:
    #     base_urls = [line.strip() for line in f]
    # urls_list
    # base_urls =['https://fda.neu.edu.vn/hoi-nghi-khoa-hoc-cong-nghe-dai-hoc-kinh-te-quoc-dan-nam-2025/']
    # ['https://nct.neu.edu.vn/', 'https://fsf.neu.edu.vn/', 'https://mfe.neu.edu.vn/', 'https://mis.neu.edu.vn/', 'https://fda.neu.edu.vn/', 'https://khoathongke.neu.edu.vn/', 'https://fit.neu.edu.vn/']
    
    print('Feeding .docx files')
    
    error_files = []  # List to store files that caused errors
    for i, file_path in enumerate(tqdm(docx_files, desc="Đang xử lý", unit="file")):
        # print(f'Reading {file_path}')
        output_json_path = f"output_{i}.json"
        
        try:
            metadata = define_metadata(file_path)
            splits = get_splits(file_path, output_json_path)
            splits_with_metadata = update_documents_metadata(splits, metadata)
            all_splits += splits_with_metadata    
        except Exception as e:
            print(f"Error processing {file_path}: {e}")
            error_files.append(file_path)  # Store the problematic file
    
    # Print all files that caused errors
    if error_files:
        print("\nFiles that caused errors:")
        for file in error_files:
            print(file)
    else:
        print("\nAll files processed successfully!")
        
    print('Feeding .json files')

# Xử lý FAQ
    # FAQ_path = "syllabus_nct_word_format/FAQ.json"
    # FAQ_splits = get_json_splits_only(FAQ_path)
    # all_splits += FAQ_splits

    

    # FAQ_path = "syllabus_nct_word_format/FAQ2.json"
    # FAQ_splits = get_json_splits_only(FAQ_path)
    # all_splits += FAQ_splits
    
    # Lưu vào vectorstore với nhúng từ Google GenAI
    # embedding = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
    print('Get embedding model /paraphrase-multilingual-MiniLM-L12-v2')
    embedding = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")   #"VoVanPhuc/sup-SimCSE-VietNamese-phobert-base")
    print('Set vectorstore FAISS')
    vectorstore = FAISS.from_documents(documents=all_splits, embedding=embedding)
    return vectorstore