Update app.py

app.py

@@ -2,30 +2,28 @@ import os,re
 import gradio as gr
 import nest_asyncio
 from langchain import PromptTemplate
-from llama_index.core import StorageContext, load_index_from_storage
+from llama_index.core import PromptTemplate, StorageContext, load_index_from_storage
+from llama_index.llms.openai import OpenAI
+
 import networkx as nx
 from pyvis.network import Network
 from IPython.display import HTML, Markdown, display
 
-# nest_asyncio.apply()
-
-kg_index_path = "./telcom_full_property_kg_processed_dynamicpath2_withpackagedata_category/" 
-kg_plot_path = kg_index_path+"full_kg.html"
 os.environ["OPENAI_API_KEY"] = os.getenv('oai')
 
 
-index = load_index_from_storage(
+#Graph-RAG 
+kg_index_path = "./telcom_full_property_kg_processed_dynamicpath2_withpackagedata_category/" 
+kg_plot_path = kg_index_path+"/full_kg.html"
+graph_rag_index = load_index_from_storage(
     StorageContext.from_defaults(persist_dir=kg_index_path)
 )
-query_engine = index.as_query_engine(
-    include_text=True,
-    similarity_top_k=20,
-)
-retriever = index.as_retriever(
-    include_text=True,  # include source text, default True
-    similarity_top_k=20,
-)
 
+#RAG 
+rag_index_path = "./telcom_RAG_full_withpackagedata_category/" 
+rag_index = load_index_from_storage(
+    StorageContext.from_defaults(persist_dir=rag_index_path)
+)
 
 teamplate_prompt_upsell = '''You are a virtual assistant for a telecom company, designed to assist users with their queries and potentially upsell services. Your task is to analyze the customer's data from context, their query, and offer the most appropriate assistance.
 
@@ -66,41 +64,70 @@ Remember to always prioritize customer satisfaction and only suggest upsells whe
 '''
 
 
+llm_eval_prompt = """You are an AI tasked with evaluating the performance of a language model (LLM) based on a given query and response. Your role is to assess the LLM's output using four specific metrics and provide scores for each.
 
+Here are the metrics you will use to evaluate the LLM's performance:
 
-def parse_response_with_regex(xml_response):
-    # Define regex patterns for each tag
-    query_type_pattern = re.compile(r'<query_type>(.*?)</query_type>', re.DOTALL)
-    answer_pattern = re.compile(r'<answer>(.*?)</answer>', re.DOTALL)
-    reference_pattern = re.compile(r'<reference>(.*?)</reference>', re.DOTALL)
-    facts_pattern = re.compile(r'<facts>(.*?)</facts>', re.DOTALL)
-    upsell_opportunity_pattern = re.compile(r'<upsell_opportunity>(.*?)</upsell_opportunity>', re.DOTALL)
+1. Comprehensiveness: How thoroughly and completely the response addresses all aspects of the query.
+2. Diversity: The variety of perspectives, examples, or approaches included in the response.
+3. Empowerment: How well the response enables the user to understand or act on the information provided.
+4. Directness: The clarity and conciseness of the response in addressing the query.
 
-    # Extract data using regex
-    query_type = query_type_pattern.search(xml_response).group(1).strip()
-    answer = answer_pattern.search(xml_response).group(1).strip()
-    reference = reference_pattern.search(xml_response).group(1).strip()
-    facts = facts_pattern.search(xml_response).group(1).strip()
-    upsell_opportunity = upsell_opportunity_pattern.search(xml_response).group(1).strip()
+To perform your evaluation, carefully analyze the following query and response:
 
-    # Format the extracted information
-    formatted_response = f"""
-    ### Query Type
-    {query_type}
+<query>
+{QUERY}
+</query>
 
-    ### Answer
-    {answer}
+<response>
+{RESPONSE}
+</response>
 
-    ### Reference
-    {reference}
+For each metric, consider the following:
 
-    ### Facts
-    {facts}
+1. Comprehensiveness: Does the response cover all aspects of the query? Are there any missing or underdeveloped points?
+2. Diversity: Does the response offer multiple viewpoints or examples? Is there a good range of information or approaches presented?
+3. Empowerment: Does the response provide actionable information or insights? Does it enhance the user's understanding or ability to address the query?
+4. Directness: Is the response clear and to the point? Does it avoid unnecessary information or tangents?
 
-    ### Upsell Opportunity
-    {upsell_opportunity}
-    """
-    return formatted_response
+Score each metric on a scale from 0 to 5, where 0 is the lowest (poor performance) and 5 is the highest (excellent performance).
+
+For each metric, provide a brief justification for your score before stating the score itself. Your justification should reference specific aspects of the query and response that influenced your decision.
+
+Present your evaluation in the following format:
+
+<evaluation>
+<metric name="Comprehensiveness">
+<justification>
+[Your justification for the Comprehensiveness score]
+</justification>
+<score>[Your score from 0-5]</score>
+</metric>
+
+<metric name="Diversity">
+<justification>
+[Your justification for the Diversity score]
+</justification>
+<score>[Your score from 0-5]</score>
+</metric>
+
+<metric name="Empowerment">
+<justification>
+[Your justification for the Empowerment score]
+</justification>
+<score>[Your score from 0-5]</score>
+</metric>
+
+<metric name="Directness">
+<justification>
+[Your justification for the Directness score]
+</justification>
+<score>[Your score from 0-5]</score>
+</metric>
+</evaluation>
+
+Ensure that your evaluation is fair, objective, and based solely on the provided query and response. Do not make assumptions about information not present in the given text.
+"""
 
 def extract_pattern_triplet(text):
     # Define the regex pattern to match the desired format
@@ -109,15 +136,61 @@ def extract_pattern_triplet(text):
     matches = pattern.findall(text)
     return "\n <br> ".join(matches)
 
-def query_tqa(query):
-    data = {
-        'QUERY': query
-    }
-    prompt = PromptTemplate(template=teamplate_prompt_upsell, input_variables=["QUERY"])
-    query_ready = prompt.format(**data)
-    response = query_engine.query(query_ready)
-    nodes = retriever.retrieve(query_ready)
-    parsed_resp = parse_response_with_regex(str(response))
+def query_rag_qa(query,search_level):
+    """
+    A function to query the RAG QA with a given query and search level. 
+    It returns the response, nodes, and response metadata. 
+    Parameters:
+    - query: The query to search for
+    - search_level: The level of similarity to search for
+    Return:
+    - response: The query response
+    - nodes: The retrieved nodes
+    - metadata: The metadata of the response
+    """
+    myretriever = rag_index.as_retriever(
+            include_text=True,  # include source text, default True
+            similarity_top_k=search_level,
+        )    
+    query_engine = rag_index.as_query_engine(
+        sub_retrievers=[
+            myretriever,
+        ],
+        include_text=True,
+        similarity_top_k=search_level,
+    )
+    response = query_engine.query(query)
+    nodes = myretriever.retrieve(query)
+
+    return response, nodes, response.metadata
+
+
+def query_graph_rag_qa(query,search_level):
+    """
+    A function to query the RAG QA with a given query and search level. 
+    It returns the response, reference, and reference text. 
+    Parameters:
+    - query: The query to search for
+    - search_level: The level of similarity to search for
+    Return:
+    - response: The query response
+    - reference: The extracted patterns
+    - reference_text: The text of the extracted patterns
+    """
+    myretriever = graph_rag_index.as_retriever(
+            include_text=True,  # include source text, default True
+            similarity_top_k=search_level,
+        )    
+    query_engine = graph_rag_index.as_query_engine(
+        sub_retrievers=[
+            myretriever,
+        ],
+        include_text=True,
+        similarity_top_k=search_level,
+    )
+    response = query_engine.query(query)
+    nodes = myretriever.retrieve(query)
+    # parsed_resp = parse_response_with_regex(str(response))
 
     reference = []
     reference_text = []
@@ -125,11 +198,36 @@ def query_tqa(query):
         reference.append(extract_pattern_triplet(node.text))
         reference_text.append(node.text)
 
-    return parsed_resp, response, reference , reference_text
+    return response, reference , reference_text
+
+def query_tqa(query,search_level):
+    grag_response, grag_reference , grag_reference_text = query_graph_rag_qa(query,search_level)
+    rag_response, rag_reference, rag_reference_text = query_rag_qa(query,search_level)
+    return grag_response, grag_reference , grag_reference_text, rag_response, rag_reference, rag_reference_text
+
+
+def eval_llm(query,rag_response,grag_response):
+    data = {'QUERY': query,
+            'RESPONSE': rag_response
+            }
+    prompt = PromptTemplate(llm_eval_prompt) 
+    query_ready = prompt.format(**data)
+    rag_eval = OpenAI().complete(query_ready)
+
+    data = {'QUERY': query,
+            'RESPONSE': grag_response
+            }
+    prompt = PromptTemplate(llm_eval_prompt) 
+    query_ready = prompt.format(**data)
+    grag_eval = OpenAI().complete(query_ready)
+    return grag_eval,rag_eval
+
 
 def plot_full_kg():
     """Plot the full knowledge graph and return the HTML representation."""
-    return HTML(filename=kg_plot_path)
+    # return HTML(filename=kg_plot_path)
+    with open(kg_plot_path, "r") as file:
+        return file.read()
 
 with gr.Blocks() as demo:
     gr.Markdown("<h1>Telcom Graph-RAG v0.1</h1>")
@@ -137,23 +235,51 @@ with gr.Blocks() as demo:
     with gr.Tab("Virtual Assistant"):
         with gr.Row():
             query_input = gr.Textbox(label="Input Your Query..")
+            search_level = gr.Slider(minimum=1, maximum=50, value=3, step=5, label="Search level")
+            ask_button = gr.Button("Ask TelcomVA!!")
         with gr.Row():
-            model_output = gr.Textbox(label="Response")
-            model_metadata = gr.Textbox(label="Raw Response")
+            with gr.Accordion("Graph-RAG!", open=True):
+                grag_output = gr.Textbox(label="Response")
+                grag_reference = gr.Textbox(label="Triplets")
+                grag_reference_text = gr.Textbox(label="Extracted Reference raw")
+
+            with gr.Accordion("RAG", open=True):
+                rag_output = gr.Textbox(label="Response")
+                rag_reference = gr.Textbox(label="Extracted Reference")
+                rag_reference_text = gr.Textbox(label="Extracted Reference raw")
         with gr.Row():
-            reference = gr.HTML(label="Extracted Reference")
-            reference_text = gr.Textbox(label="Extracted Reference raw")
+            grag_performance = gr.Textbox(label="Graph-RAG Performance")
+            rag_performance = gr.Textbox(label="RAG Performance")
+        eval_button = gr.Button("Evaluate LLMs!!")
 
 
-        ask_button = gr.Button("Ask TelcomVA!!")
 
     with gr.Accordion("Explore KG!", open=False):
-        gr.Markdown("This KG is built using a subset of Github repositories. ")
         kg_output = gr.HTML()
         plot_button = gr.Button("Plot Full KG!!")
 
-    ask_button.click(query_tqa, inputs=[query_input], outputs=[model_output,model_metadata,reference,reference_text])
+    ask_button.click(query_tqa, 
+                     inputs=[query_input,search_level], 
+                     outputs=[
+                         grag_output,
+                         grag_reference,
+                         grag_reference_text,
+                         rag_output,
+                         rag_reference,
+                         rag_reference_text
+                         ]
+    )
+    
+    eval_button.click(eval_llm,
+                      inputs=[query_input,rag_output,grag_output],
+                      outputs=[
+                          grag_performance,
+                          rag_performance
+                      ]
+    )
+
     plot_button.click(plot_full_kg, outputs=kg_output)
+
     examples = gr.Examples(
     examples=[
         ["what are the upselling ideas for roaming package you can recommend for customer Rina Wati."],
@@ -161,5 +287,6 @@ with gr.Blocks() as demo:
     inputs=[query_input]
     )
 
+
 demo.launch(auth=(os.getenv('id'), os.getenv('pass')), share=True)
 # demo.launch(share=False)