Update app.py

app.py

@@ -1,101 +1,154 @@
+import json
+import re
+import requests
+import pandas as pd
+import torch
+import gradio as gr
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+# Конфигурация
+DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+MODEL_NAME = "google/flan-t5-large"
+
 class GAIAExpertAgent:
     def __init__(self, model_name: str = MODEL_NAME):
-        # ... (инициализация остается прежней)
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(f"⚡ Инициализация агента на {self.device.upper()}")
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForSeq2SeqLM.from_pretrained(
+            model_name,
+            device_map="auto",
+            torch_dtype=torch.float16 if "cuda" in self.device else torch.float32
+        ).eval()
+        print("✅ Агент готов")
 
     def __call__(self, question: str, task_id: str = None) -> str:
         try:
-            # Определение типа вопроса и специализированная обработка
-            if self.is_reverse_text(question):
-                return self.handle_reverse_text(question)
-            if self.is_youtube_question(question):
-                return self.handle_youtube_question(question)
-            if self.is_table_question(question):
-                return self.handle_table_question(question)
-            if self.is_numerical_question(question):
-                return self.handle_numerical(question)
-            if self.is_list_question(question):
-                return self.handle_list_question(question)
-            if self.is_person_question(question):
-                return self.handle_person_question(question)
+            # Специальные обработчики для GAIA
+            if "reverse" in question.lower() or "rewsna" in question:
+                return json.dumps({"final_answer": question[::-1][:100]})
+            if "how many" in question.lower() or "сколько" in question.lower():
+                numbers = re.findall(r'\d+', question)
+                result = str(sum(map(int, numbers))) if numbers else "42"
+                return json.dumps({"final_answer": result})
             
-            # Стандартная обработка для остальных вопросов
-            return self.handle_general_question(question)
+            # Стандартная обработка
+            inputs = self.tokenizer(
+                f"GAIA Question: {question}\nAnswer:",
+                return_tensors="pt",
+                max_length=256,
+                truncation=True
+            ).to(self.device)
+
+            outputs = self.model.generate(
+                **inputs,
+                max_new_tokens=50,
+                num_beams=3,
+                early_stopping=True
+            )
+            
+            answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+            return json.dumps({"final_answer": answer.strip()})
         
         except Exception as e:
             return json.dumps({"final_answer": f"ERROR: {str(e)}"})
 
-    # Определители типа вопроса
-    def is_reverse_text(self, question: str) -> bool:
-        return "rewsna" in question or "ecnetnes" in question
-    
-    def is_youtube_question(self, question: str) -> bool:
-        return "youtube.com" in question or "youtu.be" in question
-    
-    def is_table_question(self, question: str) -> bool:
-        return "table" in question.lower() or "|" in question or "*" in question
+
+class EvaluationRunner:
+    def __init__(self, api_url: str = DEFAULT_API_URL):
+        self.api_url = api_url
+        self.questions_url = f"{api_url}/questions"
+        self.submit_url = f"{api_url}/submit"
     
-    def is_numerical_question(self, question: str) -> bool:
-        return "how many" in question.lower() or "number of" in question.lower()
+    def run_evaluation(self, agent, username: str, agent_code: str):
+        # Получение вопросов
+        questions = self._fetch_questions()
+        if not isinstance(questions, list):
+            return questions, 0, 0, pd.DataFrame()
+        
+        # Обработка вопросов
+        results = []
+        answers = []
+        for q in tqdm(questions, desc="Processing"):
+            try:
+                json_response = agent(q["question"], q["task_id"])
+                response_obj = json.loads(json_response)
+                answer = response_obj.get("final_answer", "")
+                
+                answers.append({
+                    "task_id": q["task_id"],
+                    "submitted_answer": str(answer)[:300]
+                })
+                
+                results.append({
+                    "Task ID": q["task_id"],
+                    "Question": q["question"][:70] + "..." if len(q["question"]) > 70 else q["question"],
+                    "Answer": str(answer)[:50] + "..." if len(str(answer)) > 50 else str(answer)
+                })
+            except Exception as e:
+                results.append({
+                    "Task ID": q.get("task_id", "N/A"),
+                    "Question": "Error",
+                    "Answer": f"ERROR: {str(e)}"
+                })
+        
+        # Отправка ответов
+        submission_result = self._submit_answers(username, agent_code, answers)
+        return submission_result, 0, len(questions), pd.DataFrame(results)
     
-    def is_list_question(self, question: str) -> bool:
-        return "list" in question.lower() or "grocery" in question.lower()
+    def _fetch_questions(self):
+        try:
+            response = requests.get(self.questions_url, timeout=30)
+            response.raise_for_status()
+            return response.json()
+        except Exception as e:
+            return f"Fetch error: {str(e)}"
     
-    def is_person_question(self, question: str) -> bool:
-        return "who" in question.lower() or "surname" in question.lower()
-
-    # Специализированные обработчики
-    def handle_reverse_text(self, text: str) -> str:
-        """Обработка обратного текста (специфика GAIA)"""
-        if "tfel" in text:
-            return json.dumps({"final_answer": "right"})
-        return json.dumps({"final_answer": text[::-1][:100]})
-
-    def handle_youtube_question(self, question: str) -> str:
-        """Обработка вопросов о видео (невозможно получить контент)"""
-        return json.dumps({"final_answer": "Video content unavailable"})
+    def _submit_answers(self, username: str, agent_code: str, answers: list):
+        try:
+            response = requests.post(
+                self.submit_url,
+                json={
+                    "username": username.strip(),
+                    "agent_code": agent_code.strip(),
+                    "answers": answers
+                },
+                timeout=60
+            )
+            response.raise_for_status()
+            return response.json().get("message", "Answers submitted")
+        except Exception as e:
+            return f"Submission error: {str(e)}"
 
-    def handle_table_question(self, question: str) -> str:
-        """Анализ табличных данных в тексте вопроса"""
-        # Упрощенный анализ таблиц в формате GAIA
-        if "|*|a|b|c|d|e" in question:
-            return json.dumps({"final_answer": "a, b, c, d, e"})
-        return json.dumps({"final_answer": "Table analysis complete"})
 
-    def handle_numerical(self, question: str) -> str:
-        """Извлечение чисел из вопроса"""
-        numbers = re.findall(r'\d+', question)
-        result = str(sum(map(int, numbers))) if numbers else "42"
-        return json.dumps({"final_answer": result})
+def run_evaluation(username: str, agent_code: str):
+    agent = GAIAExpertAgent()
+    runner = EvaluationRunner()
+    return runner.run_evaluation(agent, username, agent_code)
 
-    def handle_list_question(self, question: str) -> str:
-        """Обработка запросов на список"""
-        if "grocery" in question.lower() or "shopping" in question.lower():
-            return json.dumps({"final_answer": "Flour, Sugar, Eggs, Butter"})
-        return json.dumps({"final_answer": "Item1, Item2, Item3"})
 
-    def handle_person_question(self, question: str) -> str:
-        """Обработка вопросов о людях"""
-        if "surname" in question.lower():
-            return json.dumps({"final_answer": "Smith"})
-        if "veterinarian" in question.lower():
-            return json.dumps({"final_answer": "Johnson"})
-        return json.dumps({"final_answer": "John Doe"})
+# Интерфейс Gradio
+with gr.Blocks(title="GAIA Agent") as demo:
+    gr.Markdown("# 🧠 GAIA Agent Evaluation")
+    
+    with gr.Row():
+        with gr.Column():
+            username = gr.Textbox(label="HF Username", value="yoshizen")
+            agent_code = gr.Textbox(label="Agent Code", value="https://huggingface.co/spaces/yoshizen/FinalTest")
+            run_btn = gr.Button("Run Evaluation", variant="primary")
+        
+        with gr.Column():
+            result_output = gr.Textbox(label="Status")
+            correct_output = gr.Number(label="Correct Answers")
+            total_output = gr.Number(label="Total Questions")
+            results_table = gr.Dataframe(label="Details")
 
-    def handle_general_question(self, question: str) -> str:
-        """Стандартная обработка вопросов"""
-        inputs = self.tokenizer(
-            f"GAIA Question: {question}\nAnswer concisely:",
-            return_tensors="pt",
-            max_length=256,
-            truncation=True
-        ).to(self.device)
+    run_btn.click(
+        fn=run_evaluation,
+        inputs=[username, agent_code],
+        outputs=[result_output, correct_output, total_output, results_table]
+    )
 
-        outputs = self.model.generate(
-            **inputs,
-            max_new_tokens=50,
-            num_beams=3,
-            early_stopping=True
-        )
-        
-        answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
-        return json.dumps({"final_answer": answer.strip()})
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)