LamiaYT commited on
Commit
82111b7
·
1 Parent(s): 835936b
Files changed (2) hide show
  1. app.py +218 -186
  2. lang.txt +412 -0
app.py CHANGED
@@ -1,200 +1,232 @@
1
  import os
2
- import gradio as gr
3
- import requests
4
- import inspect
5
- import pandas as pd
6
- from agent import build_graph
7
-
8
- # (Keep Constants as is)
9
- # --- Constants ---
10
- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
11
-
12
- # --- Basic Agent Definition ---
13
- # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
14
- class BasicAgent:
15
- def __init__(self):
16
- print("BasicAgent initialized.")
17
- self.graph = build_graph()
18
-
19
- def __call__(self, question: str) -> str:
20
- print(f"Agent received question (first 50 chars): {question[:50]}...")
21
- # Wrap the question in a HumanMessage from langchain_core
22
- messages = [HumanMessage(content=question)]
23
- messages = self.graph.invoke({"messages": messages})
24
- answer = messages['messages'][-1].content
25
- return answer[14:]
26
-
27
-
28
- def run_and_submit_all( profile: gr.OAuthProfile | None):
29
- """
30
- Fetches all questions, runs the BasicAgent on them, submits all answers,
31
- and displays the results.
32
- """
33
- # --- Determine HF Space Runtime URL and Repo URL ---
34
- space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
35
-
36
- if profile:
37
- username= f"{profile.username}"
38
- print(f"User logged in: {username}")
39
- else:
40
- print("User not logged in.")
41
- return "Please Login to Hugging Face with the button.", None
42
-
43
- api_url = DEFAULT_API_URL
44
- questions_url = f"{api_url}/questions"
45
- submit_url = f"{api_url}/submit"
46
-
47
- # 1. Instantiate Agent ( modify this part to create your agent)
 
 
 
 
 
 
 
 
 
 
 
 
48
  try:
49
- agent = BasicAgent()
 
 
 
 
 
 
 
50
  except Exception as e:
51
- print(f"Error instantiating agent: {e}")
52
- return f"Error initializing agent: {e}", None
53
- # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
54
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
55
- print(agent_code)
56
-
57
- # 2. Fetch Questions
58
- print(f"Fetching questions from: {questions_url}")
59
  try:
60
- response = requests.get(questions_url, timeout=15)
61
- response.raise_for_status()
62
- questions_data = response.json()
63
- if not questions_data:
64
- print("Fetched questions list is empty.")
65
- return "Fetched questions list is empty or invalid format.", None
66
- print(f"Fetched {len(questions_data)} questions.")
67
- except requests.exceptions.RequestException as e:
68
- print(f"Error fetching questions: {e}")
69
- return f"Error fetching questions: {e}", None
70
- except requests.exceptions.JSONDecodeError as e:
71
- print(f"Error decoding JSON response from questions endpoint: {e}")
72
- print(f"Response text: {response.text[:500]}")
73
- return f"Error decoding server response for questions: {e}", None
74
  except Exception as e:
75
- print(f"An unexpected error occurred fetching questions: {e}")
76
- return f"An unexpected error occurred fetching questions: {e}", None
77
-
78
- # 3. Run your Agent
79
- results_log = []
80
- answers_payload = []
81
- print(f"Running agent on {len(questions_data)} questions...")
82
- for item in questions_data:
83
- task_id = item.get("task_id")
84
- question_text = item.get("question")
85
- if not task_id or question_text is None:
86
- print(f"Skipping item with missing task_id or question: {item}")
87
- continue
88
- try:
89
- submitted_answer = agent(question_text)
90
- answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
91
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
92
- except Exception as e:
93
- print(f"Error running agent on task {task_id}: {e}")
94
- results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
95
-
96
- if not answers_payload:
97
- print("Agent did not produce any answers to submit.")
98
- return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
99
-
100
- # 4. Prepare Submission
101
- submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
102
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
103
- print(status_update)
104
-
105
- # 5. Submit
106
- print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
107
  try:
108
- response = requests.post(submit_url, json=submission_data, timeout=60)
109
- response.raise_for_status()
110
- result_data = response.json()
111
- final_status = (
112
- f"Submission Successful!\n"
113
- f"User: {result_data.get('username')}\n"
114
- f"Overall Score: {result_data.get('score', 'N/A')}% "
115
- f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
116
- f"Message: {result_data.get('message', 'No message received.')}"
117
  )
118
- print("Submission successful.")
119
- results_df = pd.DataFrame(results_log)
120
- return final_status, results_df
121
- except requests.exceptions.HTTPError as e:
122
- error_detail = f"Server responded with status {e.response.status_code}."
123
- try:
124
- error_json = e.response.json()
125
- error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
126
- except requests.exceptions.JSONDecodeError:
127
- error_detail += f" Response: {e.response.text[:500]}"
128
- status_message = f"Submission Failed: {error_detail}"
129
- print(status_message)
130
- results_df = pd.DataFrame(results_log)
131
- return status_message, results_df
132
- except requests.exceptions.Timeout:
133
- status_message = "Submission Failed: The request timed out."
134
- print(status_message)
135
- results_df = pd.DataFrame(results_log)
136
- return status_message, results_df
137
- except requests.exceptions.RequestException as e:
138
- status_message = f"Submission Failed: Network error - {e}"
139
- print(status_message)
140
- results_df = pd.DataFrame(results_log)
141
- return status_message, results_df
142
  except Exception as e:
143
- status_message = f"An unexpected error occurred during submission: {e}"
144
- print(status_message)
145
- results_df = pd.DataFrame(results_log)
146
- return status_message, results_df
147
-
148
-
149
- # --- Build Gradio Interface using Blocks ---
150
- with gr.Blocks() as demo:
151
- gr.Markdown("# Basic Agent Evaluation Runner")
152
- gr.Markdown(
153
- """
154
- **Instructions:**
155
- 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
156
- 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
157
- 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
158
- ---
159
- **Disclaimers:**
160
- Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
161
- This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
162
- """
163
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
 
165
- gr.LoginButton()
166
 
167
- run_button = gr.Button("Run Evaluation & Submit All Answers")
 
168
 
169
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
170
- # Removed max_rows=10 from DataFrame constructor
171
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
 
 
 
 
172
 
173
- run_button.click(
174
- fn=run_and_submit_all,
175
- outputs=[status_output, results_table]
176
- )
 
 
177
 
178
- if __name__ == "__main__":
179
- print("\n" + "-"*30 + " App Starting " + "-"*30)
180
- # Check for SPACE_HOST and SPACE_ID at startup for information
181
- space_host_startup = os.getenv("SPACE_HOST")
182
- space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
183
-
184
- if space_host_startup:
185
- print(f"✅ SPACE_HOST found: {space_host_startup}")
186
- print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
187
- else:
188
- print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
189
-
190
- if space_id_startup: # Print repo URLs if SPACE_ID is found
191
- print(f"✅ SPACE_ID found: {space_id_startup}")
192
- print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
193
- print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
194
- else:
195
- print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
196
-
197
- print("-"*(60 + len(" App Starting ")) + "\n")
198
-
199
- print("Launching Gradio Interface for Basic Agent Evaluation...")
200
- demo.launch(debug=True, share=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
+ from dotenv import load_dotenv
3
+
4
+ # Load environment variables
5
+ load_dotenv()
6
+
7
+ # Set protobuf implementation to avoid C++ extension issues
8
+ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
9
+
10
+ # Load keys from environment
11
+ hf_token = os.getenv("HUGGINGFACE_INFERENCE_TOKEN")
12
+ serper_api_key = os.getenv("SERPER_API_KEY")
13
+
14
+ # ---- Imports ----
15
+ from langgraph.graph import START, StateGraph, MessagesState
16
+ from langgraph.prebuilt import tools_condition, ToolNode
17
+ from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint
18
+ from langchain_community.tools.tavily_search import TavilySearchResults
19
+ from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
20
+ from langchain_community.vectorstores import Chroma
21
+ from langchain_core.documents import Document
22
+ from langchain_core.messages import SystemMessage, HumanMessage
23
+ from langchain_core.tools import tool
24
+ from langchain.tools.retriever import create_retriever_tool
25
+ from langchain.embeddings import HuggingFaceEmbeddings
26
+ import json
27
+
28
+ # ---- Tools ----
29
+
30
+ @tool
31
+ def multiply(a: int, b: int) -> int:
32
+ """Multiply two numbers together."""
33
+ return a * b
34
+
35
+ @tool
36
+ def add(a: int, b: int) -> int:
37
+ """Add two numbers together."""
38
+ return a + b
39
+
40
+ @tool
41
+ def subtract(a: int, b: int) -> int:
42
+ """Subtract the second number from the first."""
43
+ return a - b
44
+
45
+ @tool
46
+ def divide(a: int, b: int) -> float:
47
+ """Divide the first number by the second. Returns float or error if dividing by zero."""
48
+ if b == 0:
49
+ raise ValueError("Cannot divide by zero.")
50
+ return a / b
51
+
52
+ @tool
53
+ def modulus(a: int, b: int) -> int:
54
+ """Returns the remainder after division of the first number by the second."""
55
+ return a % b
56
+
57
+ @tool
58
+ def wiki_search(query: str) -> str:
59
+ """Search Wikipedia for information. Useful for factual questions about people, places, events, etc."""
60
  try:
61
+ search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
62
+ formatted = "\n\n---\n\n".join(
63
+ [
64
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
65
+ for doc in search_docs
66
+ ]
67
+ )
68
+ return {"wiki_results": formatted}
69
  except Exception as e:
70
+ return f"Wikipedia search failed: {str(e)}"
71
+
72
+ @tool
73
+ def web_search(query: str) -> str:
74
+ """Search the web for current information. Useful when you need recent or non-Wikipedia information."""
 
 
 
75
  try:
76
+ search = TavilySearchResults(max_results=3)
77
+ search_docs = search.invoke(query)
78
+ formatted = "\n\n---\n\n".join(
79
+ [
80
+ f'<Document source="{doc["url"]}"/>\n{doc["content"]}\n</Document>'
81
+ for doc in search_docs
82
+ ]
83
+ )
84
+ return {"web_results": formatted}
 
 
 
 
 
85
  except Exception as e:
86
+ return f"Web search failed: {str(e)}"
87
+
88
+ @tool
89
+ def arxiv_search(query: str) -> str:
90
+ """Search academic papers on ArXiv. Useful for technical or scientific questions."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  try:
92
+ search_docs = ArxivLoader(query=query, load_max_docs=2).load()
93
+ formatted = "\n\n---\n\n".join(
94
+ [
95
+ f'<Document source="{doc.metadata["source"]}"/>\n{doc.page_content[:1000]}\n</Document>'
96
+ for doc in search_docs
97
+ ]
 
 
 
98
  )
99
+ return {"arxiv_results": formatted}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  except Exception as e:
101
+ return f"ArXiv search failed: {str(e)}"
102
+
103
+ # ---- Embedding & Vector Store Setup ----
104
+
105
+ embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
106
+
107
+ # Load QA pairs
108
+ json_QA = []
109
+ try:
110
+ with open('metadata.jsonl', 'r') as jsonl_file:
111
+ for line in jsonl_file:
112
+ json_QA.append(json.loads(line))
113
+ except Exception as e:
114
+ print(f"Error loading metadata.jsonl: {e}")
115
+ json_QA = []
116
+
117
+ documents = [
118
+ Document(
119
+ page_content=f"Question: {sample['Question']}\n\nAnswer: {sample['Final answer']}",
120
+ metadata={"source": sample["task_id"], "question": sample["Question"], "answer": sample["Final answer"]}
121
  )
122
+ for sample in json_QA
123
+ ]
124
+
125
+ try:
126
+ vector_store = Chroma.from_documents(
127
+ documents=documents,
128
+ embedding=embeddings,
129
+ persist_directory="./chroma_db",
130
+ collection_name="qa_collection"
131
+ )
132
+ vector_store.persist()
133
+ print(f"Documents inserted: {len(documents)}")
134
+ except Exception as e:
135
+ print(f"Error creating vector store: {e}")
136
+ raise
137
+
138
+ @tool
139
+ def similar_question_search(query: str) -> str:
140
+ """Search for similar questions that have been answered before. Always check here first before using other tools."""
141
+ try:
142
+ matched_docs = vector_store.similarity_search(query, k=3)
143
+ formatted = "\n\n---\n\n".join(
144
+ [
145
+ f'<Question: {doc.metadata["question"]}>\n<Answer: {doc.metadata["answer"]}>\n</Document>'
146
+ for doc in matched_docs
147
+ ]
148
+ )
149
+ return {"similar_questions": formatted}
150
+ except Exception as e:
151
+ return f"Similar question search failed: {str(e)}"
152
 
153
+ # ---- System Prompt ----
154
 
155
+ system_prompt = """
156
+ You are an expert question-answering assistant. Follow these steps for each question:
157
 
158
+ 1. FIRST check for similar questions using the similar_question_search tool
159
+ 2. If a similar question exists with a clear answer, use that answer
160
+ 3. If not, determine which tools might help answer the question
161
+ 4. Use the tools systematically to gather information
162
+ 5. Combine information from multiple sources if needed
163
+ 6. Format your final answer precisely as:
164
+ FINAL ANSWER: [your answer here]
165
 
166
+ Rules for answers:
167
+ - Numbers: plain digits only (no commas, units, or symbols)
168
+ - Strings: minimal words, no articles, full names
169
+ - Lists: comma-separated with no extra formatting
170
+ - Be concise but accurate
171
+ """
172
 
173
+ sys_msg = SystemMessage(content=system_prompt)
174
+
175
+ # ---- Tool List ----
176
+
177
+ tools = [
178
+ similar_question_search, # Check this first
179
+ multiply, add, subtract, divide, modulus, # Math tools
180
+ wiki_search, web_search, arxiv_search # Information tools
181
+ ]
182
+
183
+ # ---- Graph Definition ----
184
+
185
+ def build_graph():
186
+ try:
187
+ # Using a powerful HuggingFace model
188
+ llm = ChatHuggingFace(
189
+ llm=HuggingFaceEndpoint(
190
+ repo_id="mistralai/Mixtral-8x7B-Instruct-v0.1",
191
+ temperature=0,
192
+ max_new_tokens=512,
193
+ huggingfacehub_api_token=hf_token
194
+ )
195
+ )
196
+
197
+ llm_with_tools = llm.bind_tools(tools)
198
+
199
+ def assistant(state: MessagesState):
200
+ return {"messages": [llm_with_tools.invoke(state["messages"])]}
201
+
202
+ def retriever(state: MessagesState):
203
+ try:
204
+ # First try to find similar questions
205
+ similar = vector_store.similarity_search(state["messages"][-1].content, k=2)
206
+ if similar:
207
+ example_msg = HumanMessage(
208
+ content=f"Here are similar questions and their answers:\n\n" +
209
+ "\n\n".join([f"Q: {doc.metadata['question']}\nA: {doc.metadata['answer']}"
210
+ for doc in similar])
211
+ )
212
+ return {"messages": [sys_msg] + state["messages"] + [example_msg]}
213
+ return {"messages": [sys_msg] + state["messages"]}
214
+ except Exception as e:
215
+ print(f"Retriever error: {e}")
216
+ return {"messages": [sys_msg] + state["messages"]}
217
+
218
+ builder = StateGraph(MessagesState)
219
+ builder.add_node("retriever", retriever)
220
+ builder.add_node("assistant", assistant)
221
+ builder.add_node("tools", ToolNode(tools))
222
+
223
+ builder.add_edge(START, "retriever")
224
+ builder.add_edge("retriever", "assistant")
225
+ builder.add_conditional_edges("assistant", tools_condition)
226
+ builder.add_edge("tools", "assistant")
227
+
228
+ return builder.compile()
229
+
230
+ except Exception as e:
231
+ print(f"Error building graph: {e}")
232
+ raise
lang.txt ADDED
@@ -0,0 +1,412 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Final_Assignment_Template\app.py
2
+ import os
3
+ import gradio as gr
4
+ import requests
5
+ import inspect
6
+ import pandas as pd
7
+ from agent import build_graph
8
+
9
+ # (Keep Constants as is)
10
+ # --- Constants ---
11
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
12
+
13
+ # --- Basic Agent Definition ---
14
+ # ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
15
+ class BasicAgent:
16
+ def __init__(self):
17
+ print("BasicAgent initialized.")
18
+ self.graph = build_graph()
19
+
20
+ def __call__(self, question: str) -> str:
21
+ print(f"Agent received question (first 50 chars): {question[:50]}...")
22
+ # Wrap the question in a HumanMessage from langchain_core
23
+ messages = [HumanMessage(content=question)]
24
+ messages = self.graph.invoke({"messages": messages})
25
+ answer = messages['messages'][-1].content
26
+ return answer[14:]
27
+
28
+
29
+ def run_and_submit_all( profile: gr.OAuthProfile | None):
30
+ """
31
+ Fetches all questions, runs the BasicAgent on them, submits all answers,
32
+ and displays the results.
33
+ """
34
+ # --- Determine HF Space Runtime URL and Repo URL ---
35
+ space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
36
+
37
+ if profile:
38
+ username= f"{profile.username}"
39
+ print(f"User logged in: {username}")
40
+ else:
41
+ print("User not logged in.")
42
+ return "Please Login to Hugging Face with the button.", None
43
+
44
+ api_url = DEFAULT_API_URL
45
+ questions_url = f"{api_url}/questions"
46
+ submit_url = f"{api_url}/submit"
47
+
48
+ # 1. Instantiate Agent ( modify this part to create your agent)
49
+ try:
50
+ agent = BasicAgent()
51
+ except Exception as e:
52
+ print(f"Error instantiating agent: {e}")
53
+ return f"Error initializing agent: {e}", None
54
+ # In the case of an app running as a hugging Face space, this link points toward your codebase ( usefull for others so please keep it public)
55
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
56
+ print(agent_code)
57
+
58
+ # 2. Fetch Questions
59
+ print(f"Fetching questions from: {questions_url}")
60
+ try:
61
+ response = requests.get(questions_url, timeout=15)
62
+ response.raise_for_status()
63
+ questions_data = response.json()
64
+ if not questions_data:
65
+ print("Fetched questions list is empty.")
66
+ return "Fetched questions list is empty or invalid format.", None
67
+ print(f"Fetched {len(questions_data)} questions.")
68
+ except requests.exceptions.RequestException as e:
69
+ print(f"Error fetching questions: {e}")
70
+ return f"Error fetching questions: {e}", None
71
+ except requests.exceptions.JSONDecodeError as e:
72
+ print(f"Error decoding JSON response from questions endpoint: {e}")
73
+ print(f"Response text: {response.text[:500]}")
74
+ return f"Error decoding server response for questions: {e}", None
75
+ except Exception as e:
76
+ print(f"An unexpected error occurred fetching questions: {e}")
77
+ return f"An unexpected error occurred fetching questions: {e}", None
78
+
79
+ # 3. Run your Agent
80
+ results_log = []
81
+ answers_payload = []
82
+ print(f"Running agent on {len(questions_data)} questions...")
83
+ for item in questions_data:
84
+ task_id = item.get("task_id")
85
+ question_text = item.get("question")
86
+ if not task_id or question_text is None:
87
+ print(f"Skipping item with missing task_id or question: {item}")
88
+ continue
89
+ try:
90
+ submitted_answer = agent(question_text)
91
+ answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
92
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
93
+ except Exception as e:
94
+ print(f"Error running agent on task {task_id}: {e}")
95
+ results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
96
+
97
+ if not answers_payload:
98
+ print("Agent did not produce any answers to submit.")
99
+ return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
100
+
101
+ # 4. Prepare Submission
102
+ submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
103
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
104
+ print(status_update)
105
+
106
+ # 5. Submit
107
+ print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
108
+ try:
109
+ response = requests.post(submit_url, json=submission_data, timeout=60)
110
+ response.raise_for_status()
111
+ result_data = response.json()
112
+ final_status = (
113
+ f"Submission Successful!\n"
114
+ f"User: {result_data.get('username')}\n"
115
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
116
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
117
+ f"Message: {result_data.get('message', 'No message received.')}"
118
+ )
119
+ print("Submission successful.")
120
+ results_df = pd.DataFrame(results_log)
121
+ return final_status, results_df
122
+ except requests.exceptions.HTTPError as e:
123
+ error_detail = f"Server responded with status {e.response.status_code}."
124
+ try:
125
+ error_json = e.response.json()
126
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
127
+ except requests.exceptions.JSONDecodeError:
128
+ error_detail += f" Response: {e.response.text[:500]}"
129
+ status_message = f"Submission Failed: {error_detail}"
130
+ print(status_message)
131
+ results_df = pd.DataFrame(results_log)
132
+ return status_message, results_df
133
+ except requests.exceptions.Timeout:
134
+ status_message = "Submission Failed: The request timed out."
135
+ print(status_message)
136
+ results_df = pd.DataFrame(results_log)
137
+ return status_message, results_df
138
+ except requests.exceptions.RequestException as e:
139
+ status_message = f"Submission Failed: Network error - {e}"
140
+ print(status_message)
141
+ results_df = pd.DataFrame(results_log)
142
+ return status_message, results_df
143
+ except Exception as e:
144
+ status_message = f"An unexpected error occurred during submission: {e}"
145
+ print(status_message)
146
+ results_df = pd.DataFrame(results_log)
147
+ return status_message, results_df
148
+
149
+
150
+ # --- Build Gradio Interface using Blocks ---
151
+ with gr.Blocks() as demo:
152
+ gr.Markdown("# Basic Agent Evaluation Runner")
153
+ gr.Markdown(
154
+ """
155
+ **Instructions:**
156
+ 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ...
157
+ 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
158
+ 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
159
+ ---
160
+ **Disclaimers:**
161
+ Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions).
162
+ This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async.
163
+ """
164
+ )
165
+
166
+ gr.LoginButton()
167
+
168
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
169
+
170
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
171
+ # Removed max_rows=10 from DataFrame constructor
172
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
173
+
174
+ run_button.click(
175
+ fn=run_and_submit_all,
176
+ outputs=[status_output, results_table]
177
+ )
178
+
179
+ if __name__ == "__main__":
180
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
181
+ # Check for SPACE_HOST and SPACE_ID at startup for information
182
+ space_host_startup = os.getenv("SPACE_HOST")
183
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
184
+
185
+ if space_host_startup:
186
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
187
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
188
+ else:
189
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
190
+
191
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
192
+ print(f"✅ SPACE_ID found: {space_id_startup}")
193
+ print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
194
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
195
+ else:
196
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
197
+
198
+ print("-"*(60 + len(" App Starting ")) + "\n")
199
+
200
+ print("Launching Gradio Interface for Basic Agent Evaluation...")
201
+ demo.launch(debug=True, share=False)
202
+ Final_Assignment_Template\agent.py:
203
+ import os
204
+ from dotenv import load_dotenv
205
+
206
+ # Load environment variables
207
+ load_dotenv()
208
+
209
+ # Set protobuf implementation to avoid C++ extension issues
210
+ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
211
+
212
+ # Load keys from environment
213
+ groq_api_key = os.getenv("GROQ_API_KEY")
214
+ serper_api_key = os.getenv("SERPER_API_KEY")
215
+ hf_token = os.getenv("HUGGINGFACE_INFERENCE_TOKEN")
216
+
217
+ # ---- Imports ----
218
+ from langgraph.graph import START, StateGraph, MessagesState
219
+ from langgraph.prebuilt import tools_condition, ToolNode
220
+ from langchain_google_genai import ChatGoogleGenerativeAI
221
+ from langchain_groq import ChatGroq
222
+ from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings
223
+ from langchain_community.tools.tavily_search import TavilySearchResults
224
+ from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
225
+ from langchain_community.vectorstores import Chroma
226
+ from langchain_core.documents import Document
227
+ from langchain_core.messages import SystemMessage, HumanMessage
228
+ from langchain_core.tools import tool
229
+ from langchain.tools.retriever import create_retriever_tool
230
+ from langchain.vectorstores import Chroma
231
+ from langchain.embeddings import HuggingFaceEmbeddings
232
+ from langchain.schema import Document
233
+ import json
234
+
235
+ # ---- Tools ----
236
+
237
+ @tool
238
+ def multiply(a: int, b: int) -> int:
239
+ return a * b
240
+
241
+ @tool
242
+ def add(a: int, b: int) -> int:
243
+ return a + b
244
+
245
+ @tool
246
+ def subtract(a: int, b: int) -> int:
247
+ return a - b
248
+
249
+ @tool
250
+ def divide(a: int, b: int) -> float:
251
+ if b == 0:
252
+ raise ValueError("Cannot divide by zero.")
253
+ return a / b
254
+
255
+ @tool
256
+ def modulus(a: int, b: int) -> int:
257
+ return a % b
258
+
259
+ @tool
260
+ def wiki_search(query: str) -> str:
261
+ search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
262
+ formatted = "\n\n---\n\n".join(
263
+ [
264
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
265
+ for doc in search_docs
266
+ ]
267
+ )
268
+ return {"wiki_results": formatted}
269
+
270
+ @tool
271
+ def web_search(query: str) -> str:
272
+ search_docs = TavilySearchResults(max_results=3).invoke(query=query)
273
+ formatted = "\n\n---\n\n".join(
274
+ [
275
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
276
+ for doc in search_docs
277
+ ]
278
+ )
279
+ return {"web_results": formatted}
280
+
281
+ @tool
282
+ def arvix_search(query: str) -> str:
283
+ search_docs = ArxivLoader(query=query, load_max_docs=3).load()
284
+ formatted = "\n\n---\n\n".join(
285
+ [
286
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
287
+ for doc in search_docs
288
+ ]
289
+ )
290
+ return {"arvix_results": formatted}
291
+
292
+ # ---- Embedding & Vector Store Setup ----
293
+
294
+ embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
295
+
296
+ json_QA = []
297
+ with open('metadata.jsonl', 'r') as jsonl_file:
298
+ for line in jsonl_file:
299
+ json_QA.append(json.loads(line))
300
+
301
+ documents = [
302
+ Document(
303
+ page_content=f"Question : {sample['Question']}\n\nFinal answer : {sample['Final answer']}",
304
+ metadata={"source": sample["task_id"]}
305
+ )
306
+ for sample in json_QA
307
+ ]
308
+
309
+ vector_store = Chroma.from_documents(
310
+ documents=documents,
311
+ embedding=embeddings,
312
+ persist_directory="./chroma_db",
313
+ collection_name="my_collection"
314
+ )
315
+ vector_store.persist()
316
+ print("Documents inserted:", vector_store._collection.count())
317
+
318
+ @tool
319
+ def similar_question_search(query: str) -> str:
320
+ matched_docs = vector_store.similarity_search(query, 3)
321
+ formatted = "\n\n---\n\n".join(
322
+ [
323
+ f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
324
+ for doc in matched_docs
325
+ ]
326
+ )
327
+ return {"similar_questions": formatted}
328
+
329
+ # ---- System Prompt ----
330
+
331
+ system_prompt = """
332
+ You are a helpful assistant tasked with answering questions using a set of tools.
333
+ Now, I will ask you a question. Report your thoughts, and finish your answer with the following template:
334
+ FINAL ANSWER: [YOUR FINAL ANSWER].
335
+ YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings...
336
+ """
337
+
338
+ sys_msg = SystemMessage(content=system_prompt)
339
+
340
+ # ---- Tool List ----
341
+
342
+ tools = [
343
+ multiply, add, subtract, divide, modulus,
344
+ wiki_search, web_search, arvix_search, similar_question_search
345
+ ]
346
+
347
+ # ---- Graph Definition ----
348
+
349
+ def build_graph(provider: str = "groq"):
350
+ if provider == "groq":
351
+ llm = ChatGroq(model="qwen-qwq-32b", temperature=0, api_key=groq_api_key)
352
+ elif provider == "google":
353
+ llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0)
354
+ elif provider == "huggingface":
355
+ llm = ChatHuggingFace(
356
+ llm=HuggingFaceEndpoint(repo_id="mosaicml/mpt-30b", temperature=0)
357
+ )
358
+ else:
359
+ raise ValueError("Invalid provider: choose 'groq', 'google', or 'huggingface'.")
360
+
361
+ llm_with_tools = llm.bind_tools(tools)
362
+
363
+ def assistant(state: MessagesState):
364
+ return {"messages": [llm_with_tools.invoke(state["messages"])]}
365
+
366
+ def retriever(state: MessagesState):
367
+ similar = vector_store.similarity_search(state["messages"][0].content)
368
+ if similar:
369
+ example_msg = HumanMessage(content=f"Here is a similar question:\n\n{similar[0].page_content}")
370
+ return {"messages": [sys_msg] + state["messages"] + [example_msg]}
371
+ return {"messages": [sys_msg] + state["messages"]}
372
+
373
+ builder = StateGraph(MessagesState)
374
+ builder.add_node("retriever", retriever)
375
+ builder.add_node("assistant", assistant)
376
+ builder.add_node("tools", ToolNode(tools))
377
+ builder.add_edge(START, "retriever")
378
+ builder.add_edge("retriever", "assistant")
379
+ builder.add_conditional_edges("assistant", tools_condition)
380
+ builder.add_edge("tools", "assistant")
381
+
382
+ return builder.compile()
383
+ Final_Assignment_Template\metadata.jsonl:
384
+
385
+ Final_Assignment_Template\requirements.txt:
386
+ gradio
387
+ requests
388
+ langchain
389
+ langchain-community
390
+ langchain-core
391
+ langchain-google-genai
392
+ langchain-huggingface
393
+ langchain-groq
394
+ langchain-tavily
395
+ langchain-chroma
396
+ langgraph
397
+ sentence-transformers
398
+ huggingface_hub
399
+ supabase
400
+ arxiv
401
+ pymupdf
402
+ wikipedia
403
+ pgvector
404
+ python-dotenv
405
+ protobuf==3.20.3
406
+
407
+ Final_Assignment_Template\system_prompt.txt:
408
+ You are a helpful assistant tasked with answering questions using a set of tools.
409
+ Now, I will ask you a question. Report your thoughts, and finish your answer with the following template:
410
+ FINAL ANSWER: [YOUR FINAL ANSWER].
411
+ YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise. If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise. If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
412
+ Your answer should only start with "FINAL ANSWER: ", then follows with the answer.