app.py

import os,re
import gradio as gr
import nest_asyncio
from langchain import PromptTemplate
from llama_index.core import PromptTemplate, StorageContext, load_index_from_storage
from llama_index.llms.openai import OpenAI

import networkx as nx
from pyvis.network import Network
from IPython.display import HTML, Markdown, display

os.environ["OPENAI_API_KEY"] = os.getenv('oai')


#Graph-RAG 
kg_index_path = "./telcom_full_property_kg_processed_dynamicpath2_withpackagedata_category/" 
kg_plot_path = kg_index_path+"/full_kg.html"
graph_rag_index = load_index_from_storage(
    StorageContext.from_defaults(persist_dir=kg_index_path)
)

#RAG 
rag_index_path = "./telcom_RAG_full_withpackagedata_category/" 
rag_index = load_index_from_storage(
    StorageContext.from_defaults(persist_dir=rag_index_path)
)

teamplate_prompt_upsell = '''You are a virtual assistant for a telecom company, designed to assist users with their queries and potentially upsell services. Your task is to analyze the customer's data from context, their query, and offer the most appropriate assistance.

First, you will be given the customer's data context. This information will help you understand the customer's current plan and usage patterns:

When interacting with a customer, you will receive a query with their details like name or phone number.
<query>
{QUERY}
</query>

Analyze the query to determine the type of assistance required. Categorize it into one of the following:
1. Technical Support
2. Billing Inquiry
3. Plan Information
4. Service Upgrade
5. General Inquiry

Based on the query type and customer data, provide an appropriate response. Your response should:
1. Address the customer's immediate concern
2. Be clear and concise
3. Use a friendly and causal tone
4. Make sure to provide facts and relations for each response
5. Use Emojis to engage the customer in conversation

If the query presents an opportunity for upselling, consider recommending relevant services or upgrades based on the customer's current plan and usage patterns. However, ensure that your primary focus remains on resolving the customer's initial query.

Format your response as follows:

<response>
<query_type>[Categorized query type]</query_type>
<answer>[Your detailed response addressing the customer's query]</answer>
<reference>[Provide the reference documents used for generating the response]</reference>
<facts>[Provide the facts used for generating the response]</facts>
<upsell_opportunity>[If applicable, provide a brief upsell recommendation]</upsell_opportunity>
</response>

Remember to always prioritize customer satisfaction and only suggest upsells when they genuinely benefit the customer.
'''


llm_eval_prompt = """You are an AI tasked with evaluating the performance of a language model (LLM) based on a given query and response. Your role is to assess the LLM's output using four specific metrics and provide scores for each.

Here are the metrics you will use to evaluate the LLM's performance:

1. Comprehensiveness: How thoroughly and completely the response addresses all aspects of the query.
2. Diversity: The variety of perspectives, examples, or approaches included in the response.
3. Empowerment: How well the response enables the user to understand or act on the information provided.
4. Directness: The clarity and conciseness of the response in addressing the query.

To perform your evaluation, carefully analyze the following query and response:

<query>
{QUERY}
</query>

<response>
{RESPONSE}
</response>

For each metric, consider the following:

1. Comprehensiveness: Does the response cover all aspects of the query? Are there any missing or underdeveloped points?
2. Diversity: Does the response offer multiple viewpoints or examples? Is there a good range of information or approaches presented?
3. Empowerment: Does the response provide actionable information or insights? Does it enhance the user's understanding or ability to address the query?
4. Directness: Is the response clear and to the point? Does it avoid unnecessary information or tangents?

Score each metric on a scale from 0 to 5, where 0 is the lowest (poor performance) and 5 is the highest (excellent performance).

For each metric, provide a brief justification for your score before stating the score itself. Your justification should reference specific aspects of the query and response that influenced your decision.

Present your evaluation in the following format:

<evaluation>
<metric name="Comprehensiveness">
<justification>
[Your justification for the Comprehensiveness score]
</justification>
<score>[Your score from 0-5]</score>
</metric>

<metric name="Diversity">
<justification>
[Your justification for the Diversity score]
</justification>
<score>[Your score from 0-5]</score>
</metric>

<metric name="Empowerment">
<justification>
[Your justification for the Empowerment score]
</justification>
<score>[Your score from 0-5]</score>
</metric>

<metric name="Directness">
<justification>
[Your justification for the Directness score]
</justification>
<score>[Your score from 0-5]</score>
</metric>
</evaluation>

Ensure that your evaluation is fair, objective, and based solely on the provided query and response. Do not make assumptions about information not present in the given text.
"""

def extract_pattern_triplet(text):
    # Define the regex pattern to match the desired format
    pattern = re.compile(r'\b\w+\b\s*->\s*\b\w+\b\s*->\s*\b\w+\b')
    # Find all matches in the text
    matches = pattern.findall(text)
    return "\n <br> ".join(matches)

def query_rag_qa(query,search_level):
    """
    A function to query the RAG QA with a given query and search level. 
    It returns the response, nodes, and response metadata. 
    Parameters:
    - query: The query to search for
    - search_level: The level of similarity to search for
    Return:
    - response: The query response
    - nodes: The retrieved nodes
    - metadata: The metadata of the response
    """
    myretriever = rag_index.as_retriever(
            include_text=True,  # include source text, default True
            similarity_top_k=search_level,
        )    
    query_engine = rag_index.as_query_engine(
        sub_retrievers=[
            myretriever,
        ],
        include_text=True,
        similarity_top_k=search_level,
    )
    response = query_engine.query(query)
    nodes = myretriever.retrieve(query)

    return response, nodes, response.metadata


def query_graph_rag_qa(query,search_level):
    """
    A function to query the RAG QA with a given query and search level. 
    It returns the response, reference, and reference text. 
    Parameters:
    - query: The query to search for
    - search_level: The level of similarity to search for
    Return:
    - response: The query response
    - reference: The extracted patterns
    - reference_text: The text of the extracted patterns
    """
    myretriever = graph_rag_index.as_retriever(
            include_text=True,  # include source text, default True
            similarity_top_k=search_level,
        )    
    query_engine = graph_rag_index.as_query_engine(
        sub_retrievers=[
            myretriever,
        ],
        include_text=True,
        similarity_top_k=search_level,
    )
    response = query_engine.query(query)
    nodes = myretriever.retrieve(query)
    # parsed_resp = parse_response_with_regex(str(response))

    reference = []
    reference_text = []
    for node in nodes:
        reference.append(extract_pattern_triplet(node.text))
        reference_text.append(node.text)

    return response, reference , reference_text

def query_tqa(query,search_level):
    grag_response, grag_reference , grag_reference_text = query_graph_rag_qa(query,search_level)
    rag_response, rag_reference, rag_reference_text = query_rag_qa(query,search_level)
    return grag_response, grag_reference , grag_reference_text, rag_response, rag_reference, rag_reference_text


def eval_llm(query,rag_response,grag_response):
    data = {'QUERY': query,
            'RESPONSE': rag_response
            }
    prompt = PromptTemplate(llm_eval_prompt) 
    query_ready = prompt.format(**data)
    rag_eval = OpenAI().complete(query_ready)

    data = {'QUERY': query,
            'RESPONSE': grag_response
            }
    prompt = PromptTemplate(llm_eval_prompt) 
    query_ready = prompt.format(**data)
    grag_eval = OpenAI().complete(query_ready)
    return grag_eval,rag_eval


def plot_full_kg():
    """Plot the full knowledge graph and return the HTML representation."""
    # return HTML(filename=kg_plot_path)
    with open(kg_plot_path, "r") as file:
        return file.read()

with gr.Blocks() as demo:
    gr.Markdown("<h1>Telcom Graph-RAG v0.1</h1>")

    with gr.Tab("Virtual Assistant"):
        with gr.Row():
            query_input = gr.Textbox(label="Input Your Query..")
            search_level = gr.Slider(minimum=1, maximum=50, value=3, step=5, label="Search level")
            ask_button = gr.Button("Ask TelcomVA!!")
        with gr.Row():
            with gr.Accordion("Graph-RAG!", open=True):
                grag_output = gr.Textbox(label="Response")
                grag_reference = gr.Textbox(label="Triplets")
                grag_reference_text = gr.Textbox(label="Extracted Reference raw")

            with gr.Accordion("RAG", open=True):
                rag_output = gr.Textbox(label="Response")
                rag_reference = gr.Textbox(label="Extracted Reference")
                rag_reference_text = gr.Textbox(label="Extracted Reference raw")
        with gr.Row():
            grag_performance = gr.Textbox(label="Graph-RAG Performance")
            rag_performance = gr.Textbox(label="RAG Performance")
        eval_button = gr.Button("Evaluate LLMs!!")



    with gr.Accordion("Explore KG!", open=False):
        kg_output = gr.HTML()
        plot_button = gr.Button("Plot Full KG!!")

    ask_button.click(query_tqa, 
                     inputs=[query_input,search_level], 
                     outputs=[
                         grag_output,
                         grag_reference,
                         grag_reference_text,
                         rag_output,
                         rag_reference,
                         rag_reference_text
                         ]
    )
    
    eval_button.click(eval_llm,
                      inputs=[query_input,rag_output,grag_output],
                      outputs=[
                          grag_performance,
                          rag_performance
                      ]
    )

    plot_button.click(plot_full_kg, outputs=kg_output)

    examples = gr.Examples(
    examples=[
        ["what are the upselling ideas for roaming package you can recommend for customer Rina Wati."],
    ],
    inputs=[query_input]
    )


demo.launch(auth=(os.getenv('id'), os.getenv('pass')), share=True)
# demo.launch(share=False)