Added Gradio UI

app.py

@@ -0,0 +1,71 @@
+import gradio as gr
+import logging
+from generator.compute_rmse_auc_roc_metrics import compute_rmse_auc_roc_metrics
+
+def launch_gradio(vector_store, dataset, gen_llm, val_llm):
+    """
+    Launch the Gradio app with pre-initialized objects.
+    """
+    def answer_question_with_metrics(query):
+        try:
+            logging.info(f"Processing query: {query}")
+            
+            # Generate metrics using the passed objects
+            from main import generate_metrics
+            response, metrics = generate_metrics(gen_llm, val_llm, vector_store, query, 1)
+            
+            response_text = f"Response: {response}\n\n"
+            metrics_text = "Metrics:\n"
+            for key, value in metrics.items():
+                if key != 'response':
+                    metrics_text += f"{key}: {value}\n"
+            
+            return response_text, metrics_text
+        except Exception as e:
+            logging.error(f"Error processing query: {e}")
+            return f"An error occurred: {e}"
+
+    def compute_and_display_metrics():
+        try:
+            # Call the function to compute metrics
+            relevance_rmse, utilization_rmse, adherence_auc = compute_rmse_auc_roc_metrics(
+                gen_llm, val_llm, dataset, vector_store, 10
+            )
+            
+            # Format the result for display
+            result = (
+                f"Relevance RMSE Score: {relevance_rmse}\n"
+                f"Utilization RMSE Score: {utilization_rmse}\n"
+                f"Overall Adherence AUC-ROC: {adherence_auc}\n"
+            )
+            return result
+        except Exception as e:
+            logging.error(f"Error during metrics computation: {e}")
+            return f"An error occurred: {e}"
+
+    # Define Gradio Blocks layout
+    with gr.Blocks() as interface:
+        interface.title = "Real Time RAG Pipeline Q&A"
+        gr.Markdown("### Real Time RAG Pipeline Q&A")  # Heading
+        gr.Markdown("Ask a question and get a response with metrics calculated from the RAG pipeline.")  # Description
+        
+        with gr.Row():
+            query_input = gr.Textbox(label="Ask a question", placeholder="Type your query here")
+        with gr.Row():
+            clear_query_button = gr.Button("Clear")  # Clear button
+            submit_button = gr.Button("Submit", variant="primary") # Submit button
+        with gr.Row():
+            answer_output = gr.Textbox(label="Response", placeholder="Response will appear here")
+        with gr.Row():
+            metrics_output = gr.Textbox(label="Metrics", placeholder="Metrics will appear here")
+        with gr.Row():
+            compute_rmse_button = gr.Button("Compute RMSE & AU-ROC", variant="primary")
+            rmse_output = gr.Textbox(label="RMSE & AU-ROC Score", placeholder="RMSE & AU-ROC score will appear here")
+    
+        
+        # Define button actions
+        submit_button.click(fn=answer_question_with_metrics, inputs=[query_input], outputs=[answer_output, metrics_output])
+        clear_query_button.click(fn=lambda: "", outputs=[query_input])  # Clear query input 
+        compute_rmse_button.click(fn=compute_and_display_metrics, outputs=[rmse_output])
+
+    interface.launch()

generator/compute_rmse_auc_roc_metrics.py

@@ -25,7 +25,7 @@ def compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, num_qu
         query = document['question']
         logging.info(f'Query number: {i + 1}')
         # Call the generate_metrics for each query
-        metrics = generate_metrics(gen_llm, val_llm, vector_store, query)
+        response, metrics = generate_metrics(gen_llm, val_llm, vector_store, query, 15)
         
         # Extract predicted metrics (ensure these are continuous if possible)
         predicted_relevance = metrics.get('Context Relevance', 0) if metrics else 0
@@ -69,3 +69,5 @@ def compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, num_qu
     logging.info(f"Relevance RMSE score: {relevance_rmse}")
     logging.info(f"Utilization RMSE score: {utilization_rmse}")
     logging.info(f"Overall Adherence AUC-ROC: {adherence_auc}")
+
+    return relevance_rmse, utilization_rmse, adherence_auc

generator/generate_metrics.py

@@ -5,7 +5,7 @@ from retriever.retrieve_documents import retrieve_top_k_documents
 from generator.compute_metrics import get_metrics
 from generator.extract_attributes import extract_attributes
 
-def generate_metrics(gen_llm, val_llm, vector_store, query):
+def generate_metrics(gen_llm, val_llm, vector_store, query, time_to_wait):
     logging.info(f'Query: {query}')
     
     # Step 1: Retrieve relevant documents for given query
@@ -22,7 +22,7 @@ def generate_metrics(gen_llm, val_llm, vector_store, query):
     logging.info(f"Response from LLM: {response}")
 
     # Add a sleep interval to avoid hitting the rate limit
-    time.sleep(25)  # Adjust the sleep time as needed
+    time.sleep(time_to_wait)  # Adjust the sleep time as needed
     
     # Step 3: Extract attributes and total sentences for each query
     logging.info(f"Extracting attributes through validation LLM")
@@ -32,4 +32,4 @@ def generate_metrics(gen_llm, val_llm, vector_store, query):
     # Step 4 : Call the get metrics calculate metrics
     metrics = get_metrics(attributes, total_sentences)
 
-    return metrics
+    return response, metrics

main.py

@@ -6,6 +6,7 @@ from retriever.embed_documents import embed_documents
 from generator.generate_metrics import generate_metrics
 from generator.initialize_llm import initialize_generation_llm
 from generator.initialize_llm import initialize_validation_llm
+from app import launch_gradio
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -36,15 +37,18 @@ def main():
     val_llm = initialize_validation_llm()
 
     # Sample question
-    row_num = 2
-    query = dataset[row_num]['question']
+    #row_num = 30
+    #query = dataset[row_num]['question']
 
     # Call generate_metrics for above sample question
     #generate_metrics(gen_llm, val_llm, vector_store, query)
     
     #Compute RMSE and AUC-ROC for entire dataset
-    compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, 10)
+    #compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, 10)
     
+    # Launch the Gradio app
+    launch_gradio(vector_store, dataset, gen_llm, val_llm)
+
     logging.info("Finished!!!")
 
 if __name__ == "__main__":

requirements.txt

@@ -7,4 +7,5 @@ langchain
 llama-index
 langchain-community
 langchain_groq
-langchain-huggingface
+langchain-huggingface
+gradio