Update app.py

app.py

@@ -7,10 +7,68 @@ from langchain_community.vectorstores import FAISS
 from langchain.chains import RetrievalQA
 from langchain_huggingface.embeddings import HuggingFaceEmbeddings
 from langchain.prompts import PromptTemplate 
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_text_splitters import CharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+import os
+import logging
+logging.getLogger("langchain.text_splitter").setLevel(logging.ERROR)
+import warnings
+warnings.filterwarnings("ignore")
+from langchain_community.document_loaders import RecursiveUrlLoader
+import yaml
+
+
+
+# ------------ criando vs -----------------
+
+## knowledge base offline
+url_list = [
+"https://www.infinitepay.io", 
+"https://www.infinitepay.io/maquininha", 
+"https://www.infinitepay.io/maquininha-celular", 
+"https://www.infinitepay.io/tap-to-pay", 
+"https://www.infinitepay.io/pdv", 
+"https://www.infinitepay.io/receba-na-hora", 
+"https://www.infinitepay.io/gestao-de-cobranca",  
+"https://www.infinitepay.io/gestao-de-cobranca-2",  
+"https://www.infinitepay.io/link-de-pagamento", 
+"https://www.infinitepay.io/loja-online", 
+"https://www.infinitepay.io/boleto", 
+"https://www.infinitepay.io/conta-digital", 
+"https://www.infinitepay.io/conta-pj", 
+"https://www.infinitepay.io/pix",
+"https://www.infinitepay.io/pix-parcelado", 
+"https://www.infinitepay.io/emprestimo", 
+"https://www.infinitepay.io/cartao", 
+"https://www.infinitepay.io/rendimento",
+'https://www.infinitepay.io/taxas',
+'https://www.cloudwalk.io/',
+'https://www.cloudwalk.io/#our-mission',
+'https://www.cloudwalk.io/#our-pillars',
+'https://www.cloudwalk.io/#our-products',
+]
+
+# Carregue o conteúdo da página web como documentos LangChain
+loader = WebBaseLoader(web_paths=url_list_unique)
+docs = loader.load()
+print(f"Total de páginas carregadas: {len(docs)}")
+
+text_splitter = CharacterTextSplitter(chunk_size=1500, chunk_overlap=100)
+split_docs = text_splitter.split_documents(docs)
+embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL, 
+                                   cache_folder=cache_folder)
+vector_store = FAISS.from_documents(split_docs, embeddings)
+# vs_base = "../vector_store/vs_base"
+os.makedirs(VS_BASE, exist_ok=True)
+vector_store.save_local(VS_BASE)
+print(f"vs_base salva em {VS_BASE}")
 
 # --- CONFIGURAÇÕES DE MODELOS ---
-# LLM_MODEL = 'google/gemma-3-4b-it'
-LLM_MODEL = 'google/gemma-3-1b-it'
+LLM_MODEL = 'google/gemma-3-4b-it'
+# LLM_MODEL = 'google/gemma-3-1b-it'
 EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 
 # --- CONFIGURAÇÃO DO TOKEN HF ---