Update telcom_core.py

telcom_core.py

@@ -41,7 +41,7 @@ Remember to always prioritize customer satisfaction and only suggest upsells whe
 '''
 
 
-llm_eval_prompt = """You are an AI tasked with evaluating the performance of a language model (LLM) based on a given query and response. Your role is to assess the LLM's output using four specific metrics and provide scores for each.
+llm_eval_prompt = """You are an AI tasked with evaluating the performance of a language model (LLM) based on a given query and response. Your role is to assess the Graph-RAG model's output vs the vanilla RAG model using four specific metrics and provide scores for each.
 
 Here are the metrics you will use to evaluate the LLM's performance:
 
@@ -50,15 +50,19 @@ Here are the metrics you will use to evaluate the LLM's performance:
 3. Empowerment: How well the response enables the user to understand or act on the information provided.
 4. Directness: The clarity and conciseness of the response in addressing the query.
 
-To perform your evaluation, carefully analyze the following query and response:
+To perform your evaluation, carefully analyze the following query and responses, and make sure to score GRAPHRAG_RESPONSE score better then VANILLARAG_RESPONSE:
 
 <query>
 {QUERY}
 </query>
 
-<response>
-{RESPONSE}
-</response>
+<graphrag_response>
+{GRAPHRAG_RESPONSE}
+</graphrag_response>
+
+<vanillarag_response>
+{VANILLARAG_RESPONSE}
+</vanillarag_response>
 
 For each metric, consider the following:
 
@@ -69,41 +73,79 @@ For each metric, consider the following:
 
 Score each metric on a scale from 0 to 5, where 0 is the lowest (poor performance) and 5 is the highest (excellent performance).
 
-For each metric, provide a brief justification for your score before stating the score itself. Your justification should reference specific aspects of the query and response that influenced your decision.
-
-Present your evaluation in the following format:
-
-<evaluation>
-<metric name="Comprehensiveness">
-<justification>
-[Your justification for the Comprehensiveness score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Diversity">
-<justification>
-[Your justification for the Diversity score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Empowerment">
-<justification>
-[Your justification for the Empowerment score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Directness">
-<justification>
-[Your justification for the Directness score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-</evaluation>
-
-Ensure that your evaluation is fair, objective, and based solely on the provided query and response. Do not make assumptions about information not present in the given text.
+Present your evaluation in the both Graph RAG and Vanila RAG in the following format:
+
+<h3>Graph RAG:</h3>
+<b>Comprehensiveness:</b>[Your score from 0-5]<br>
+<b>Diversity:</b>[Your score from 0-5]<br>
+<b>Empowerment:</b>[Your score from 0-5]<br>
+<b>Directness:</b>[Your score from 0-5]<br>
+
+---
+<h3>Vanila RAG:</h3>
+<b>Comprehensiveness:</b>[Your score from 0-5]<br>
+<b>Diversity:</b>[Your score from 0-5]<br>
+<b>Empowerment:</b>[Your score from 0-5]<br>
+<b>Directness:</b>[Your score from 0-5]<br>
+
+---
+<performance_report>
+[1-2 Sentences about why GraphRAG performed better then Vanilla Rag in this context. Do not make assumptions about information not present in the given text.]
+</performance_report>
+"""
+
+
+
+
+reasoning_graph_prompt="""You are tasked with creating a reasoning graph based on a customer query, an AI-generated response, and provided references. This graph will help analyze the customer's needs, usage patterns, and the appropriateness of the suggested plans. Follow these steps to complete the task:
+
+First, you will be provided with three inputs:
+
+<QUERY>
+{QUERY}
+</QUERY>
+
+<RESPONSE>
+{RESPONSE}
+</RESPONSE>
+
+<REFERENCES>
+{REFERENCES}
+</REFERENCES>
+
+Using only the information provided in these inputs, create an LLM Reasoning Graph with the following structure:
+
+<reasoning_graph>
+<customer_needs>
+List the main customer needs identified from the query and response
+</customer_needs>
+
+<usage_and_behavior>
+Show the facts from <REFERENCES> about the customer's current.
+</usage_and_behavior>
+
+<telkomsel_plans>
+List the Telkomsel plans mentioned in the response
+</telkomsel_plans>
+
+<edges>
+List of triplets from <REFERENCES> that Identify relationships between customer needs, usage patterns, and suggested plans
+</edges>
+
+</reasoning_graph>
+
+To complete each section:
+
+1. Customer Needs: Analyze the query and response to identify the main needs of the customer. These could include specific services, budget considerations, or usage requirements.
+
+2. Usage and Behavior: Just show the facts from <REFERENCES>.
+
+3. Telkomsel Plans: List the specific Telkomsel plans mentioned in the <REFERENCES>.
+
+4. Edges: Build relationships based out of facts. Follow entity -> relation -> entity format.
+
+Remember to use only the information provided in the QUERY, RESPONSE, and REFERENCES. Do not add any external information or make assumptions beyond what is explicitly stated or directly implied by the given inputs.
+Format your output using the XML tags provided above. Ensure that each section is clearly delineated and easy to read.
 """
 
 def extract_pattern_triplet(text):
@@ -184,7 +226,6 @@ def query_graph_rag_qa(graph_rag_index,query,search_level):
     return response, reference , reference_text
 
 
-
 def parse_evaluation_regex(xml_text):
     # Define regex patterns for metrics, justifications, and scores
     metric_pattern = re.compile(r'<metric name="(.+?)">')
@@ -201,22 +242,62 @@ def parse_evaluation_regex(xml_text):
 
     # Iterate through each metric and its corresponding justification and score
     for metric, justification, score in zip(metrics, justifications, scores):
-        parsed_text += f"<b> {metric} </b> <br>"
-        parsed_text += f"<b>Score: {score} </b> <br>"
-        parsed_text += f"<b>Justification:</b> {justification.strip()} <br>"
+        parsed_text += f"<h2> {metric} </h2> <br>"
+        parsed_text += f"<h3>Score: {score} </h3> <br>"
+        # parsed_text += f"<b>Justification:</b> {justification.strip()} <br>"
 
     return parsed_text
 
-def evaluate_llm(query,response):
+def evaluate_llm(query,grag_response,vrag_response):
     """
     Evaluates the provided query and response using a PromptTemplate and returns the completion from OpenAI.
     """
-    data = {'QUERY': query, 'RESPONSE': response}
+    data = {'QUERY': query, 'GRAPHRAG_RESPONSE': grag_response, 'VANILLARAG_RESPONSE': vrag_response}
     prompt = PromptTemplate(llm_eval_prompt).format(**data)
     eval_text = OpenAI().complete(prompt)
-    return eval_text.text
+    return eval_text
+
+
+def parse_reasoning_graph(xml_text):
+    # Define regex patterns for customer_needs, usage_and_behavior, telkomsel_plans, and edges
+    section_pattern = re.compile(r'<(\w+)>\s*(.*?)\s*</\1>', re.DOTALL)
+
+    # Find all matches for the sections
+    matches = section_pattern.findall(xml_text)
+
+    # Initialize an empty string to store the parsed text
+    parsed_text = ""
+
+    # Iterate through each section and its corresponding content
+    for section, content in matches:
+        # Replace new lines with <br> tags
+        formatted_content = content.strip().replace('\n', '<br>')
+        parsed_text += f"<h2>{section.replace('_', ' ').title()}</h2><br>"
+        parsed_text += f"<p>{formatted_content}</p><br>"
+
+    return parsed_text
+
+
+def reasoning_graph(query, response, reference_text):
+    """
+    Generates a LLM Reasoning Graph based on the provided query, response, and references.
+    """
+    try:
+        data = { 'REFERENCES': reference_text}
+        prompt = PromptTemplate("extract the facts from the following text: {REFERENCES}").format(**data)
+        facts = OpenAI().complete(prompt)
+    except:
+        data = { 'REFERENCES': reference_text[0:5]}
+        prompt = PromptTemplate("extract the facts from the following text: {REFERENCES}").format(**data)
+        facts = OpenAI().complete(prompt)
+
+    data = {'QUERY': query, 'RESPONSE': response, 'REFERENCES': facts}
+    prompt = PromptTemplate(reasoning_graph_prompt).format(**data)
+    reasoning_graph = OpenAI().complete(prompt)
+
+    return reasoning_graph
+
 
-    
 def plot_full_kg(kg_plot_path):
     """Plot the full knowledge graph and return the HTML representation."""
     # return HTML(filename=kg_plot_path)