import sys import re import os from typing import List # from .sonnet_eval import sonnet_errors from .execute_code import execute_code_with_timeout def clean_output_for_arithmetic(output: str) -> str: """ Clean the output for arithmetic problems. Args: output (str): The output to clean. Returns: str: The cleaned output. """ if "=" in output: output = output.split("=")[1].strip() if " is" in output: output = output.split(" is")[1].strip() if " equals" in output: output = output.split(" equals")[1].strip() if " evaluates to" in output: output = output.split(" evaluates to")[1].strip() if " is equal to" in output: output = output.split(" is equal to")[1].strip() return output def clean_output_for_GameOf24(output: str) -> str: """ Clean the output for GameOf24 problems. """ if "=" in output: output = output.split("=")[0].strip() if "is" in output: output = output.split("is")[1].strip() if "equals" in output: output = output.split("equals")[0].strip() if "evaluates to" in output: output = output.split("evaluates to")[0].strip() return output def eval_for_GameOf24(input: str, output: str) -> bool: """ Given an input and output, check if the output is correct and follows the rules of the game. """ clean_output = output clean_output = clean_output_for_GameOf24(output) clean_output = clean_output.replace("x", "*").strip() clean_output = clean_output.replace("×", "*").strip() clean_output = clean_output.replace("÷", "/").strip() try: # Get the value of the expression using eval value = eval(clean_output) if not (abs(value - 24) < 1e-3): return False # Split the input and output digits by space input_digits = input.split(" ") # Replace the following symbols with space replacements = ["+", "-", "*", "/", "÷", "(", ")"] for symbol in replacements: clean_output = clean_output.replace(symbol, " ") # Replace multiple spaces with single space clean_output = re.sub(" +", " ", clean_output) clean_output = clean_output.strip() output_digits = clean_output.split(" ") # Sort the digits input_digits.sort() output_digits.sort() # Check if the digits are the same if input_digits != output_digits: return False return True except Exception as e: return False def remove_punctuation(output: str) -> str: """ Remove punctuation from the output. """ markers = [",", ";", ":", ".", '"'] for marker in markers: output = output.replace(marker, "") return output def convert_newline_to_space(output: str) -> str: """ Convert newline to space. """ output = output.replace("\n", " ") return output def eval_for_exact_matching_with_no_punctuation( output: str, target: str ) -> bool: """ Evaluate if the output is exactly the same as the target. """ output = remove_punctuation(output) output = convert_newline_to_space(output) if target == output: return True return False def eval_for_softmatch(input: str, output: str, target: str) -> bool: """ Evaluate if the output is a soft match of the target. """ output = remove_punctuation(output) if target in output: return True return False def eval_for_CheckmateInOne(input: str, output: str, target: str) -> bool: """ Evaluate if the output is a checkmate in one. """ output = output.strip() if output[-1] == "#": output = output.split(" ")[-1].strip() # Based on the input, determine the number of the last move last_move = input.split(".")[-1].strip() move_idx = input.split(".")[-2].split(" ")[-1].strip() # If the last move is an empty string, then the last move is white; otherwise, it is black if last_move == "": last_move = "White" else: last_move = "Black" next_move_idx = str(int(move_idx) + 1) if not (next_move_idx in output): if target in output or (target[1] == 'x' and (target[0] + target[2:]) in output): return True else: output = output.split(next_move_idx)[0].strip() if target in output or (target[1] == 'x' and (target[0] + target[2:]) in output): return True return False def eval_equation_balancer(input: str, output: str, target: str) -> bool: """ Evaluate if the output is a valid equation balancer. """ output = output.split("=")[0].strip() target_val = target.split("=")[1].strip() target = target.split("=")[0].strip() # First make sure that the output has the same format as the target (when operators (e.g., +, -, *, /) are removed) output_nums = output.replace("+", "").replace("-", "").replace("*", "").replace("/", "").replace(" ", "").strip() target_nums = target.replace("+", "").replace("-", "").replace("*", "").replace("/", "").replace(" ", "").strip() if output_nums != target_nums: return False # Now, evaluate the output and target try: output_value = eval(output) if abs(output_value - eval(target_val)) < 1e-6: return True except Exception as e: return False return False def eval_for_multiple_choice(input_text: str, final_answer: str, target: str) -> bool: """ Evaluates if the final answer matches the target using pattern matching. Args: input_text (str): The original question text including options final_answer (str): The model's answer target (str): The correct answer Returns: bool: True if answer is correct, False otherwise """ # Handle empty or None inputs if not final_answer or not target: return False def clean_text(text: str) -> str: if not text: return "" return text.lower().strip().replace('`', '').replace('(', '').replace(')', '') def extract_option_text(input_text: str, option_letter: str) -> str: try: # Try different formats of options sections options_section = "" if 'options:' in input_text.lower(): options_section = input_text.lower().split('options:')[1].strip() elif 'choices:' in input_text.lower(): options_section = input_text.lower().split('choices:')[1].strip() if not options_section: # Try to find options in the format (A) text, (B) text lines = input_text.lower().split('\n') for i, line in enumerate(lines): if line.strip().startswith(f'({option_letter})') or line.strip().startswith(f'{option_letter})'): return line.split(')', 1)[1].strip() # Process the options section if found for line in options_section.split('\n'): line = line.strip() if line.startswith(f'({option_letter})') or line.startswith(f'{option_letter})'): return line.split(')', 1)[1].strip() # Handle options like "A. text" format if line.startswith(f'{option_letter}.'): return line.split('.', 1)[1].strip() except: return '' return '' # Full option match (A), (B), etc. (e.g., (A) == (A)) if final_answer == target: return True # Clean and normalize inputs clean_answer = clean_text(final_answer) clean_target = clean_text(target) # Handle target formats: (A), A), A, etc. target_letter = "" if len(clean_target) == 1: target_letter = clean_target elif clean_target.endswith(')'): target_letter = clean_target[-2] else: # Extract the last character if it's a letter a-d or A-D last_char = clean_target[-1] if last_char in 'abcd': target_letter = last_char # Direct letter match (a, b, c, d) if len(clean_answer) == 1 and clean_answer in 'abcd' and clean_answer == target_letter: return True # Handle answer formats like "A" or "A." if clean_answer.startswith(target_letter) and (len(clean_answer) == 1 or (len(clean_answer) == 2 and clean_answer[1] == '.')): return True # Handle answer formats like "Option A" or "Answer is A" if clean_answer.endswith(target_letter) and (clean_answer[-2:] == f" {target_letter}" or clean_answer[-3:] == f" {target_letter}."): return True # Text content match - check if the target option text is in the answer target_text = extract_option_text(input_text, target_letter) if target_text and target_text in clean_answer: return True # Handle numerical answers (if target is a number and answer contains that number) if target_letter.isdigit() and target_letter in clean_answer: return True return False def eval_for_pyton_programming_puzzles(input: str, output: str) -> bool: """ Evaluate if the output is a valid Python programming puzzle solution. """ if "```python" in output: output = output.split("```python")[-1].strip() output = output.split("```")[0].strip() if "def sat" in output: if "from typing" not in output: output = f"from typing import *\n{output}" code = f"{output}\nanswer = solution()\nprint(sat(answer))" else: code = f"from typing import *\n{input}\n{output}\nanswer = solution()\nprint(sat(answer))" code = code.replace("List[", "list[") eval_bool = execute_code_with_timeout(code, timeout=3) if "NameError: name 'answer' is not defined" in eval_bool: print(f"Eval bool: {eval_bool}") print(f"Code:\n{code}") print("*" * 100) if "True" in eval_bool: return True return False