Update qa_txt.py

qa_txt.py

@@ -11,21 +11,21 @@ import tqdm
 from langchain_community.vectorstores import FAISS
 import accelerate
 
-c_splitter = CharacterTextSplitter(
+# c_splitter = CharacterTextSplitter(
     
-    chunk_size = 350,
-    chunk_overlap = 4,
-    separator = """,
-      ]""",
+#     chunk_size = 350,
+#     chunk_overlap = 4,
+#     separator = """,
+#       ]""",
 
-)
+# )
 
-def load_doc(file_path):
-    loader = TextLoader(file_path)
-    pages = loader.load()
-    text_splitter = c_splitter
-    doc_splits = text_splitter.split_documents(pages)
-    return doc_splits
+# def load_doc(file_path):
+#     loader = TextLoader(file_path)
+#     pages = loader.load()
+#     text_splitter = c_splitter
+#     doc_splits = text_splitter.split_documents(pages)
+#     return doc_splits
 
 
 
@@ -36,13 +36,13 @@ llm = HuggingFaceEndpoint(
             top_k = 30,
         )
 
-def process_data():
-  splt = load_doc('intents_v2.txt')
-  embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
-  db = FAISS.from_documents(splt, embeddings)
-  return db
+# def process_data():
+#   splt = load_doc('intents_v2.txt')
+#   embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
+#   db = FAISS.from_documents(splt, embeddings)
+#   return db
 
-db = process_data()
+# db = process_data()
 
 
 prompt = """Vous êtes un formateur de modèles GPT. Vous excellez dans l'affinement des modèles GPT, la lecture et l'analyse de documents ainsi que l'association de balises aux entrées utilisateur appropriées.