cleanup

app.py

@@ -9,226 +9,57 @@ import networkx as nx
 from pyvis.network import Network
 from IPython.display import HTML, Markdown, display
 
-os.environ["OPENAI_API_KEY"] = os.getenv('oai')
 
+# Define constants for index paths
+os.environ["OPENAI_API_KEY"] = os.getenv('oai')
+KG_INDEX_PATH = "./telcom_full_property_kg_processed_dynamicpath2_withpackagedata_category/"
+KG_PLOT_PATH = KG_INDEX_PATH + "/full_kg.html"
+RAG_INDEX_PATH = "./telcom_RAG_full_withpackagedata_category/"
 
-#Graph-RAG 
-kg_index_path = "./telcom_full_property_kg_processed_dynamicpath2_withpackagedata_category/" 
-kg_plot_path = kg_index_path+"/full_kg.html"
+# Load Graph-RAG index
 graph_rag_index = load_index_from_storage(
-    StorageContext.from_defaults(persist_dir=kg_index_path)
+    StorageContext.from_defaults(persist_dir=KG_INDEX_PATH)
 )
 
-#RAG 
-rag_index_path = "./telcom_RAG_full_withpackagedata_category/" 
+# Load RAG index
 rag_index = load_index_from_storage(
-    StorageContext.from_defaults(persist_dir=rag_index_path)
+    StorageContext.from_defaults(persist_dir=RAG_INDEX_PATH)
 )
 
-teamplate_prompt_upsell = '''You are a virtual assistant for a telecom company, designed to assist users with their queries and potentially upsell services. Your task is to analyze the customer's data from context, their query, and offer the most appropriate assistance.
-
-First, you will be given the customer's data context. This information will help you understand the customer's current plan and usage patterns:
-
-When interacting with a customer, you will receive a query with their details like name or phone number.
-<query>
-{QUERY}
-</query>
-
-Analyze the query to determine the type of assistance required. Categorize it into one of the following:
-1. Technical Support
-2. Billing Inquiry
-3. Plan Information
-4. Service Upgrade
-5. General Inquiry
-
-Based on the query type and customer data, provide an appropriate response. Your response should:
-1. Address the customer's immediate concern
-2. Be clear and concise
-3. Use a friendly and causal tone
-4. Make sure to provide facts and relations for each response
-5. Use Emojis to engage the customer in conversation
-
-If the query presents an opportunity for upselling, consider recommending relevant services or upgrades based on the customer's current plan and usage patterns. However, ensure that your primary focus remains on resolving the customer's initial query.
-
-Format your response as follows:
-
-<response>
-<query_type>[Categorized query type]</query_type>
-<answer>[Your detailed response addressing the customer's query]</answer>
-<reference>[Provide the reference documents used for generating the response]</reference>
-<facts>[Provide the facts used for generating the response]</facts>
-<upsell_opportunity>[If applicable, provide a brief upsell recommendation]</upsell_opportunity>
-</response>
-
-Remember to always prioritize customer satisfaction and only suggest upsells when they genuinely benefit the customer.
-'''
-
-
-llm_eval_prompt = """You are an AI tasked with evaluating the performance of a language model (LLM) based on a given query and response. Your role is to assess the LLM's output using four specific metrics and provide scores for each.
-
-Here are the metrics you will use to evaluate the LLM's performance:
-
-1. Comprehensiveness: How thoroughly and completely the response addresses all aspects of the query.
-2. Diversity: The variety of perspectives, examples, or approaches included in the response.
-3. Empowerment: How well the response enables the user to understand or act on the information provided.
-4. Directness: The clarity and conciseness of the response in addressing the query.
-
-To perform your evaluation, carefully analyze the following query and response:
-
-<query>
-{QUERY}
-</query>
-
-<response>
-{RESPONSE}
-</response>
-
-For each metric, consider the following:
-
-1. Comprehensiveness: Does the response cover all aspects of the query? Are there any missing or underdeveloped points?
-2. Diversity: Does the response offer multiple viewpoints or examples? Is there a good range of information or approaches presented?
-3. Empowerment: Does the response provide actionable information or insights? Does it enhance the user's understanding or ability to address the query?
-4. Directness: Is the response clear and to the point? Does it avoid unnecessary information or tangents?
-
-Score each metric on a scale from 0 to 5, where 0 is the lowest (poor performance) and 5 is the highest (excellent performance).
-
-For each metric, provide a brief justification for your score before stating the score itself. Your justification should reference specific aspects of the query and response that influenced your decision.
-
-Present your evaluation in the following format:
-
-<evaluation>
-<metric name="Comprehensiveness">
-<justification>
-[Your justification for the Comprehensiveness score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Diversity">
-<justification>
-[Your justification for the Diversity score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Empowerment">
-<justification>
-[Your justification for the Empowerment score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-
-<metric name="Directness">
-<justification>
-[Your justification for the Directness score]
-</justification>
-<score>[Your score from 0-5]</score>
-</metric>
-</evaluation>
-
-Ensure that your evaluation is fair, objective, and based solely on the provided query and response. Do not make assumptions about information not present in the given text.
-"""
-
-def extract_pattern_triplet(text):
-    # Define the regex pattern to match the desired format
-    pattern = re.compile(r'\b\w+\b\s*->\s*\b\w+\b\s*->\s*\b\w+\b')
-    # Find all matches in the text
-    matches = pattern.findall(text)
-    return "\n <br> ".join(matches)
-
-def query_rag_qa(query,search_level):
-    """
-    A function to query the RAG QA with a given query and search level. 
-    It returns the response, nodes, and response metadata. 
-    Parameters:
-    - query: The query to search for
-    - search_level: The level of similarity to search for
-    Return:
-    - response: The query response
-    - nodes: The retrieved nodes
-    - metadata: The metadata of the response
+def query_tqa(query, search_level):
     """
-    myretriever = rag_index.as_retriever(
-            include_text=True,  # include source text, default True
-            similarity_top_k=search_level,
-        )    
-    query_engine = rag_index.as_query_engine(
-        sub_retrievers=[
-            myretriever,
-        ],
-        include_text=True,
-        similarity_top_k=search_level,
-    )
-    response = query_engine.query(query)
-    nodes = myretriever.retrieve(query)
+    Query both Graph-RAG and RAG models and return their responses and references.
 
-    return response, nodes, response.metadata
+    Args:
+        query (str): The query to be processed.
+        search_level (str): The level of search to be performed.
 
-
-def query_graph_rag_qa(query,search_level):
-    """
-    A function to query the RAG QA with a given query and search level. 
-    It returns the response, reference, and reference text. 
-    Parameters:
-    - query: The query to search for
-    - search_level: The level of similarity to search for
-    Return:
-    - response: The query response
-    - reference: The extracted patterns
-    - reference_text: The text of the extracted patterns
+    Returns:
+        tuple: Responses and references from both Graph-RAG and RAG models.
     """
-    myretriever = graph_rag_index.as_retriever(
-            include_text=True,  # include source text, default True
-            similarity_top_k=search_level,
-        )    
-    query_engine = graph_rag_index.as_query_engine(
-        sub_retrievers=[
-            myretriever,
-        ],
-        include_text=True,
-        similarity_top_k=search_level,
-    )
-    response = query_engine.query(query)
-    nodes = myretriever.retrieve(query)
-    # parsed_resp = parse_response_with_regex(str(response))
-
-    reference = []
-    reference_text = []
-    for node in nodes:
-        reference.append(extract_pattern_triplet(node.text))
-        reference_text.append(node.text)
-
-    return response, reference , reference_text
+    grag_response, grag_reference, grag_reference_text = query_graph_rag_qa(query, search_level)
+    rag_response, rag_reference, rag_reference_text = query_rag_qa(query, search_level)
+    return grag_response, grag_reference, grag_reference_text, rag_response, rag_reference, rag_reference_text
 
-def query_tqa(query,search_level):
-    grag_response, grag_reference , grag_reference_text = query_graph_rag_qa(query,search_level)
-    rag_response, rag_reference, rag_reference_text = query_rag_qa(query,search_level)
-    return grag_response, grag_reference , grag_reference_text, rag_response, rag_reference, rag_reference_text
 
+def eval_llm(query, rag_response, grag_response):
+    """
+    Evaluate responses from both Graph-RAG and RAG models using an LLM.
 
-def eval_llm(query,rag_response,grag_response):
-    data = {'QUERY': query,
-            'RESPONSE': rag_response
-            }
-    prompt = PromptTemplate(llm_eval_prompt) 
-    query_ready = prompt.format(**data)
-    rag_eval = OpenAI().complete(query_ready)
-
-    data = {'QUERY': query,
-            'RESPONSE': grag_response
-            }
-    prompt = PromptTemplate(llm_eval_prompt) 
-    query_ready = prompt.format(**data)
-    grag_eval = OpenAI().complete(query_ready)
-    return grag_eval,rag_eval
+    Args:
+        query (str): The query that was used to generate the responses.
+        rag_response (str): The response from the RAG model.
+        grag_response (str): The response from the Graph-RAG model.
 
+    Returns:
+        tuple: Evaluation results for both responses.
+    """
+    grag_eval = evaluate_llm(query, grag_response)
+    rag_eval = evaluate_llm(query, rag_response)
+    return grag_eval, rag_eval
 
-def plot_full_kg():
-    """Plot the full knowledge graph and return the HTML representation."""
-    # return HTML(filename=kg_plot_path)
-    with open(kg_plot_path, "r") as file:
-        return file.read()
 
+ 
 with gr.Blocks() as demo:
     gr.Markdown("<h1>Telcom Graph-RAG v0.1</h1>")
 
@@ -237,6 +68,8 @@ with gr.Blocks() as demo:
             query_input = gr.Textbox(label="Input Your Query..")
             search_level = gr.Slider(minimum=1, maximum=50, value=3, step=5, label="Search level")
             ask_button = gr.Button("Ask TelcomVA!!")
+
+            
         with gr.Row():
             with gr.Accordion("Graph-RAG!", open=True):
                 grag_output = gr.Textbox(label="Response")
@@ -247,6 +80,8 @@ with gr.Blocks() as demo:
                 rag_output = gr.Textbox(label="Response")
                 rag_reference = gr.Textbox(label="Extracted Reference")
                 rag_reference_text = gr.Textbox(label="Extracted Reference raw")
+
+
         with gr.Row():
             grag_performance = gr.Textbox(label="Graph-RAG Performance")
             rag_performance = gr.Textbox(label="RAG Performance")
@@ -286,7 +121,4 @@ with gr.Blocks() as demo:
     ],
     inputs=[query_input]
     )
-
-
 demo.launch(auth=(os.getenv('id'), os.getenv('pass')), share=True)
-# demo.launch(share=False)