update

README.md

@@ -10,4 +10,75 @@ pinned: false
 license: mit
 ---
 
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+# TinyV
+
+This Hugging Face Space hosts an Answer Verification Tool (TinyV) powered by the `zhangchenxu/TinyV-1.5B` model. The tool is designed specifically for RL training to verify if a model's answer is semantically equivalent to a ground truth answer.
+
+## What This Tool Does
+
+The Answer Verification Tool analyzes:
+- A question
+- A ground truth answer
+- A model-generated answer
+
+It then determines if the model's answer is correct, even if there are minor discrepancies in formatting or wording.
+
+The verification is LLM-based rather than exact matching, which helps reduce false negatives in evaluation pipelines.
+
+## How to Use
+
+### Web Interface
+
+1. Enter the question in the first box
+2. Enter the ground truth answer
+3. Enter the model's answer to verify
+4. Adjust model parameters if needed (optional)
+5. Click "Verify Answer" to see the result
+
+The tool will return:
+- **True** if the model answer is correct
+- **False** if the model answer is incorrect
+
+### API Usage
+
+You can also use this tool via API:
+
+```python
+from gradio_client import Client
+
+client = Client("zhangchenxu/TinyV")
+result = client.predict(
+    question="What is the capital of France?",
+    ground_truth="The capital of France is Paris.",
+    model_answer="Paris is the capital of France.",
+    temperature=0.3,
+    top_p=0.95,
+    max_tokens=128,
+    api_name="/verify"
+)
+print(result)
+```
+
+## Advanced Settings
+
+- **Temperature**: Controls randomness. Lower values make output more deterministic (default: 0.3)
+- **Top-p**: Controls diversity via nucleus sampling (default: 0.95)
+- **Max Tokens**: Maximum tokens to generate in response (default: 128)
+
+## Model Information
+
+This tool uses the `zhangchenxu/TinyV-1.5B` model, which has been optimized for answer verification tasks.
+
+The model uses the following prompt template:
+
+```
+You are an AI tasked with identifying false negatives in answer verification. A false negative occurs when a model's answer is essentially correct but is marked as incorrect due to minor discrepancies or formatting issues. Your job is to analyze the given question, ground truth answer, and model answer to determine if the model's answer is actually correct despite appearing different from the ground truth.
+
+<question>{question}</question>
+
+<ground_truth_answer>{ground_truth}</ground_truth_answer>
+
+<model_answer>{model_answer}</model_answer>
+
+Return "True" if the model's answer is correct, otherwise return "False".
+```

app.py

@@ -1,64 +1,192 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
+import time
 
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+# Initialize the client with your model
+client = InferenceClient("zhangchenxu/TinyV-1.5B")
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+# The prompt template for the LLM verifier
+LV_PROMPT = """
+You are an AI tasked with identifying false negatives in answer verification. A false negative occurs when a model's answer is essentially correct but is marked as incorrect due to minor discrepancies or formatting issues. Your job is to analyze the given question, ground truth answer, and model answer to determine if the model's answer is actually correct despite appearing different from the ground truth.
 
-    messages.append({"role": "user", "content": message})
+<question>{question}</question>
 
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
+<ground_truth_answer>{ground_truth}</ground_truth_answer>
 
+<model_answer>{model_answer}</model_answer>
 
+Return "True" if the model's answer is correct, otherwise return "False".
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
 
+# Main verification function
+def verify_answer(question, ground_truth, model_answer, temperature, top_p, max_tokens):
+    # Format the prompt with user inputs
+    prompt = LV_PROMPT.format(
+        question=question,
+        ground_truth=ground_truth,
+        model_answer=model_answer
+    )
+    
+    # Prepare the message format required by the API
+    messages = [
+        {"role": "system", "content": "You are a helpful AI assistant that verifies answers."},
+        {"role": "user", "content": prompt}
+    ]
+    
+    # Initialize response
+    response_text = ""
+    
+    try:
+        # Stream the response for better UX
+        for message in client.chat_completion(
+            messages,
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
+        ):
+            token = message.choices[0].delta.content
+            if token:
+                response_text += token
+                yield response_text
+    except Exception as e:
+        yield f"Error: {str(e)}"
+
+# Create the Gradio interface
+with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), title="Answer Verification Tool") as demo:
+    # Header with title and description
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Image("https://huggingface.co/front/assets/huggingface_logo.svg", scale=1, show_label=False, height=64)
+        with gr.Column(scale=5):
+            gr.Markdown(
+                """
+                # Answer Verification Tool
+                
+                This tool verifies if an answer is correct compared to a ground truth answer, even if there are minor differences in formatting or wording.
+                """
+            )
+    
+    # Main interface
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ## How to Use
+                
+                1. Enter the question in the first box
+                2. Enter the ground truth answer
+                3. Enter the model's answer to verify
+                4. Adjust model parameters if needed
+                5. Click "Verify Answer" to see the result
+                
+                ### What this tool does
+                
+                This tool determines if a model's answer is semantically correct compared to a ground truth answer, even if there are minor discrepancies in formatting or wording.
+                
+                The model analyzes both answers and returns:
+                - **True** if the model answer is correct
+                - **False** if the model answer is incorrect
+                
+                ### API Usage Example
+                ```python
+                from gradio_client import Client
+                
+                client = Client("zhangchenxu/TinyV")
+                result = client.predict(
+                    question="What is the capital of France?",
+                    ground_truth="The capital of France is Paris.",
+                    model_answer="Paris is the capital of France.",
+                    temperature=0.3,
+                    top_p=0.95,
+                    max_tokens=128,
+                    api_name="/verify"
+                )
+                print(result)
+                ```
+                """
+            )
+            
+            # Model parameters (hidden in a collapsible section)
+            with gr.Accordion("Advanced Settings", open=False):
+                temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.3, step=0.1, label="Temperature")
+                top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
+                max_tokens = gr.Slider(minimum=32, maximum=512, value=128, step=32, label="Max Tokens")
+                
+        with gr.Column(scale=1):
+            with gr.Box():
+                gr.Markdown("## Input")
+                question = gr.Textbox(lines=3, label="Question", placeholder="Enter the question here...")
+                ground_truth = gr.Textbox(lines=5, label="Ground Truth Answer", placeholder="Enter the correct answer here...")
+                model_answer = gr.Textbox(lines=5, label="Model Answer", placeholder="Enter the answer to verify here...")
+                
+                verify_btn = gr.Button("Verify Answer", variant="primary")
+                
+                gr.Markdown("## Result")
+                result = gr.Textbox(label="Verification Result", placeholder="Result will appear here...", lines=5)
+                
+    # Connect the interface to the verification function
+    verify_btn.click(
+        verify_answer,
+        inputs=[question, ground_truth, model_answer, temperature, top_p, max_tokens],
+        outputs=result
+    )
+    
+    # Examples
+    with gr.Accordion("Examples", open=True):
+        gr.Examples(
+            examples=[
+                [
+                    "What is the capital of France?",
+                    "The capital of France is Paris.",
+                    "Paris is the capital of France.",
+                    0.3,
+                    0.95,
+                    128,
+                ],
+                [
+                    "What is 2+2?",
+                    "4",
+                    "The answer is 4.",
+                    0.3,
+                    0.95,
+                    128,
+                ],
+                [
+                    "When was the Declaration of Independence signed?",
+                    "July 4, 1776",
+                    "The Declaration of Independence was signed on July 4th, 1776.",
+                    0.3,
+                    0.95,
+                    128,
+                ],
+                [
+                    "List the first three planets from the sun.",
+                    "Mercury, Venus, Earth",
+                    "The first three planets from the sun are Mercury, Venus, and Earth.",
+                    0.3,
+                    0.95,
+                    128,
+                ],
+            ],
+            inputs=[question, ground_truth, model_answer, temperature, top_p, max_tokens],
+            outputs=result,
+        )
+    
+    # Add footer with extra information
+    with gr.Row():
+        gr.Markdown(
+            """
+            ### About
+            This tool uses the zhangchenxu/TinyV-1.5B model to verify answers.
+            
+            The verification is based on semantic similarity rather than exact matching,
+            allowing for different phrasings and formats of the same correct answer.
+            """
+        )
 
+# Define the public API
+demo.queue()
+# Launch the app
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()