Update processing.py

processing.py

@@ -1,15 +1,17 @@
+# processing.py
+
 from langchain.schema import HumanMessage
 from output_parser import attachment_parser, bigfive_parser, personality_parser
-from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import OpenAIEmbeddings
 from langchain_community.vectorstores import FAISS
 from llm_loader import load_model
 from config import openai_api_key
 from langchain.chains import RetrievalQA
-import json
 import os
+import json
 
 # Initialize embeddings and FAISS index
-embedding_model = OpenAIEmbeddings(model="text-embedding-3-large", openai_api_key=openai_api_key)
+embedding_model = OpenAIEmbeddings(openai_api_key=openai_api_key)
 
 # Path to the knowledge files
 knowledge_files = {
@@ -28,18 +30,11 @@ for key, file_path in knowledge_files.items():
 # Create a FAISS index from the knowledge documents
 faiss_index = FAISS.from_texts(documents, embedding_model)
 
-# Save FAISS index locally (optional, in case you want to persist it)
-faiss_index.save_local("faiss_index")
-
-# If you want to load the FAISS index later, use this:
-# faiss_index = FAISS.load_local("faiss_index", embedding_model)
-
-# Load the LLM using llm_loader.py
-llm = load_model(openai_api_key)  # Load the model using your custom loader
+# Load the LLM
+llm = load_model(openai_api_key)
 
 # Initialize the retrieval chain
-retriever = faiss_index.as_retriever()
-qa_chain = RetrievalQA.from_llm(llm=llm, retriever=retriever)
+qa_chain = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=faiss_index.as_retriever())
 
 def load_text(file_path: str) -> str:
     with open(file_path, 'r', encoding='utf-8') as file:
@@ -59,7 +54,15 @@ def process_task(llm, input_text: str, general_task: str, specific_task: str, ou
 
     # Perform retrieval to get the most relevant context
     relevant_docs = qa_chain({"query": truncated_input})
-    retrieved_knowledge = "\n".join([doc.page_content for doc in relevant_docs['documents']])
+    
+    # Print the structure of relevant_docs for debugging
+    print("Structure of relevant_docs:", json.dumps(relevant_docs, indent=2, default=str))
+
+    # Extract the retrieved knowledge
+    if isinstance(relevant_docs, dict) and 'result' in relevant_docs:
+        retrieved_knowledge = relevant_docs['result']
+    else:
+        retrieved_knowledge = str(relevant_docs)
 
     # Combine the retrieved knowledge with the original prompt
     prompt = f"""{general_task}
@@ -111,4 +114,4 @@ def process_input(input_text: str, llm):
                     results[speaker_id] = {}
                 results[speaker_id][task_name] = speaker_result
 
-    return results
+    return results