Upload 12 files

app.py

@@ -0,0 +1,173 @@
+import os
+# Ensure SAMBANOVA_BASE_URL is in the environment for litellm
+# This should be set before dynamic_cheatsheet.language_model is imported if it relies on it at import time,
+# but it's generally used at runtime when making the API call.
+# Setting it here early in app.py is a safeguard.
+SAMBANOVA_DEFINED_BASE_URL = "https://api.sambanova.ai/v1"
+if "SAMBANOVA_BASE_URL" not in os.environ:
+    os.environ["SAMBANOVA_BASE_URL"] = SAMBANOVA_DEFINED_BASE_URL
+    print(f"SAMBANOVA_BASE_URL environment variable set to: {SAMBANOVA_DEFINED_BASE_URL}")
+elif os.environ["SAMBANOVA_BASE_URL"] != SAMBANOVA_DEFINED_BASE_URL:
+    print(f"Warning: SAMBANOVA_BASE_URL environment variable is already set to {os.environ['SAMBANOVA_BASE_URL']}, but app expects {SAMBANOVA_DEFINED_BASE_URL}. Using the existing one.")
+
+import gradio as gr
+import sys
+
+# Add the project root to the Python path to allow importing dynamic_cheatsheet
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".")))
+
+from dynamic_cheatsheet.language_model import LanguageModel
+
+# --- Configuration ---
+SAMBANOVA_API_KEY = os.environ.get("SAMBANOVA_API_KEY")
+# SAMBANOVA_BASE_URL is now set from SAMBANOVA_DEFINED_BASE_URL to env var if not present
+SAMBANOVA_MODEL_NAME = "samba/DeepSeek-R1-Distill-Llama-70B" # Using litellm convention for SambaNova
+
+GENERATOR_PROMPT_PATH = "prompts/generator_prompt.txt"
+CURATOR_PROMPT_PATH = "prompts/curator_prompt_for_dc_cumulative.txt"
+
+GENERATOR_PROMPT = ""
+CURATOR_PROMPT = ""
+
+try:
+    with open(GENERATOR_PROMPT_PATH, "r") as f:
+        GENERATOR_PROMPT = f.read()
+    with open(CURATOR_PROMPT_PATH, "r") as f:
+        CURATOR_PROMPT = f.read()
+except FileNotFoundError:
+    print(f"Error: Prompt files not found at {GENERATOR_PROMPT_PATH} or {CURATOR_PROMPT_PATH}. Please ensure they exist.")
+    GENERATOR_PROMPT = "You are a helpful assistant. Given a question and a cheatsheet, provide an answer. Cheatsheet: [[CHEATSHEET]] Question: [[QUESTION]] FINAL ANSWER: <answer></answer>"
+    CURATOR_PROMPT = "You are a helpful assistant. Given a question, a model answer, and a previous cheatsheet, update the cheatsheet. Previous Cheatsheet: [[PREVIOUS_CHEATSHEET]] Question: [[QUESTION]] Model Answer: [[MODEL_ANSWER]] NEW CHEATSHEET: <cheatsheet></cheatsheet>"
+
+# --- Global variable for cheatsheet ---
+current_cheatsheet_cache = "(empty)"
+
+def initialize_model():
+    if not SAMBANOVA_API_KEY:
+        raise gr.Error("SAMBANOVA_API_KEY environment variable not set. Please set it in your Hugging Face Space secrets or local environment.")
+    # LanguageModel will be modified to handle samba/ prefix using env vars for API key/base URL via litellm
+    model = LanguageModel(
+        model_name=SAMBANOVA_MODEL_NAME
+    )
+    return model
+
+def generate_cheatsheet_func(training_data_text, progress=gr.Progress(track_tqdm=True)):
+    global current_cheatsheet_cache
+    if not training_data_text.strip():
+        current_cheatsheet_cache = "(empty)"
+        return "Training data is empty. Cheatsheet reset to (empty)."
+    
+    model = initialize_model()
+    
+    training_examples = [ex.strip() for ex in training_data_text.split("\n") if ex.strip()]
+    
+    cheatsheet_content = "(empty)"
+    
+    progress(0, desc="Initializing Cheatsheet Generation")
+    for i, example_input in enumerate(progress.tqdm(training_examples, desc="Generating Cheatsheet")):
+        print(f"Processing training example {i+1}/{len(training_examples)}: {example_input[:50]}...")
+        try:
+            results_dict = model.advanced_generate(
+                approach_name="DynamicCheatsheet_Cumulative",
+                input_txt=example_input,
+                cheatsheet=cheatsheet_content, 
+                generator_template=GENERATOR_PROMPT,
+                cheatsheet_template=CURATOR_PROMPT,
+                temperature=0.1,
+                max_tokens=1024
+            )
+            cheatsheet_content = results_dict.get("final_cheatsheet", cheatsheet_content)
+        except Exception as e:
+            print(f"Error processing example '{example_input[:50]}...': {e}")
+            # Continue with the current cheatsheet, and show error in UI
+            gr.Warning(f"Error on example '{example_input[:30]}...': {e}. Skipping this example.")
+            pass       
+    current_cheatsheet_cache = cheatsheet_content
+    return current_cheatsheet_cache
+
+def get_answers_func(user_query):
+    global current_cheatsheet_cache
+    if not user_query.strip():
+        return "Query is empty.", "Query is empty."
+
+    model = initialize_model()
+    answer_with_cheatsheet = "Error retrieving answer."
+    answer_without_cheatsheet = "Error retrieving answer."
+
+    # Inference WITH cheatsheet
+    try:
+        print(f"Querying WITH cheatsheet ({current_cheatsheet_cache[:50]}...)")
+        results_with_cheatsheet = model.advanced_generate(
+            approach_name="DynamicCheatsheet_Cumulative",
+            input_txt=user_query,
+            cheatsheet=current_cheatsheet_cache,
+            generator_template=GENERATOR_PROMPT,
+            cheatsheet_template=CURATOR_PROMPT, 
+            temperature=0.1,
+            max_tokens=512
+        )
+        answer_with_cheatsheet = results_with_cheatsheet.get("final_answer", "Error: Could not extract answer.")
+    except Exception as e:
+        print(f"Error (with cheatsheet): {e}")
+        answer_with_cheatsheet = f"Error during inference with cheatsheet: {e}"
+
+    # Inference WITHOUT cheatsheet
+    try:
+        print(f"Querying WITHOUT cheatsheet...")
+        results_without_cheatsheet = model.advanced_generate(
+            approach_name="DynamicCheatsheet_Cumulative", 
+            input_txt=user_query,
+            cheatsheet="(empty)", 
+            generator_template=GENERATOR_PROMPT,
+            cheatsheet_template=CURATOR_PROMPT,
+            temperature=0.1,
+            max_tokens=512
+        )
+        answer_without_cheatsheet = results_without_cheatsheet.get("final_answer", "Error: Could not extract answer.")
+    except Exception as e:
+        print(f"Error (without cheatsheet): {e}")
+        answer_without_cheatsheet = f"Error during inference without cheatsheet: {e}"
+        
+    return answer_with_cheatsheet, answer_without_cheatsheet
+
+# --- Gradio Interface ---
+with gr.Blocks(title="Task Caching Demo", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# Task Caching Demo")
+    gr.Markdown("Demonstrates the effect of using a dynamically generated cheatsheet (Task Caching) on model inference. Uses SambaNova API via `litellm`.")
+
+    with gr.Tabs():
+        with gr.TabItem("1. Generate Cheatsheet (Task Caching)"):
+            gr.Markdown("Paste your training data below, one example per line. This data will be used to build a cumulative cheatsheet. The process may take some time depending on the number of examples.")
+            training_data_input = gr.Textbox(lines=10, label="Training Data", placeholder="Example 1: What is the capital of France?\nExample 2: Solve 2+2.")
+            generate_cheatsheet_button = gr.Button("Generate Cheatsheet (Task Caching)", variant="primary")
+            cheatsheet_output = gr.Textbox(label="Generated Cheatsheet", lines=15, interactive=False, show_label=True)
+            generate_cheatsheet_button.click(
+                fn=generate_cheatsheet_func, 
+                inputs=training_data_input, 
+                outputs=cheatsheet_output,
+                show_progress="full"
+            )
+
+        with gr.TabItem("2. Test Inference"):
+            gr.Markdown("Enter your query below. The model will attempt to answer it twice: once using the generated cheatsheet (if any), and once without it.")
+            query_input = gr.Textbox(lines=3, label="Your Query", placeholder="e.g., What is the solution to 5 6 6 8 in the Game of 24?")
+            get_answers_button = gr.Button("Get Answers", variant="primary")
+            
+            with gr.Row():
+                answer_with_cheatsheet_output = gr.Textbox(label="Answer WITH Cheatsheet", lines=10, interactive=False, show_label=True)
+                answer_without_cheatsheet_output = gr.Textbox(label="Answer WITHOUT Cheatsheet", lines=10, interactive=False, show_label=True)
+            
+            get_answers_button.click(
+                fn=get_answers_func, 
+                inputs=query_input, 
+                outputs=[answer_with_cheatsheet_output, answer_without_cheatsheet_output]
+            )
+    
+    gr.Markdown("**Important:** Ensure `SAMBANOVA_API_KEY` is set as a secret in your Hugging Face Space or as an environment variable if running locally. `SAMBANOVA_BASE_URL` is set to `https://api.sambanova.ai/v1` by default if not found in environment.")
+
+if __name__ == "__main__":
+    if not SAMBANOVA_API_KEY:
+        print("Warning: SAMBANOVA_API_KEY is not set. The application will likely fail to contact the SambaNova API.")
+        print("Please set the SAMBANOVA_API_KEY environment variable.")
+    demo.launch()
+

dynamic_cheatsheet/language_model.py

@@ -0,0 +1,298 @@
+import numpy as np
+import tiktoken
+from typing import List, Tuple
+from sklearn.metrics.pairwise import cosine_similarity
+from .utils.execute_code import extract_and_run_python_code
+from .utils.extractor import extract_answer, extract_cheatsheet
+from litellm import completion
+from functools import partial
+import os # Added for SAMBANOVA env vars
+
+class LanguageModel:
+    def __init__(self,
+        model_name: str,
+    ) -> None:
+        """
+        LanguageModel class to interact with different language models.
+
+        Arguments:
+            model_name : str : The name of the language model to use.
+
+        Raises:
+            ValueError : If the model name is not found or supported.
+        """
+
+        self.model_name = model_name
+
+        # Known model list (remains the same)
+        known_model_list = [
+            "openai/gpt-4o-mini", "openai/gpt-4o-mini-2024-07-18",
+            "openai/gpt-4o", "openai/gpt-4o-2024-08-06", "openai/gpt-4o-2024-11-20",
+            "openai/gpt-3.5-turbo",
+            "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo",
+            "meta-llama/Llama-3.3-70B-Instruct-Turbo",
+            "openai/o3-mini", "openai/o3-mini-2025-01-31",
+            "openai/o1", "openai/o1-2024-12-17",
+            "anthropic/claude-3-5-sonnet-latest", "anthropic/claude-3-5-sonnet-20241022",
+            "anthropic/claude-3-5-haiku-latest", "anthropic/claude-3-5-haiku-20241022",
+            "anthropic/claude-3-7-sonnet-latest", "anthropic/claude-3-7-sonnet-20250219",
+            "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
+            "together_ai/deepseek-ai/DeepSeek-R1",
+            "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
+            "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
+            "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct",
+            "together_ai/Qwen/QwQ-32B",
+            "together_ai/Qwen/Qwen2-72B-Instruct",
+            "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
+            "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo",
+            "gemini/gemini-2.0-flash",
+            "ollama/llama3:70b",
+        ]
+
+        # Load the client for the model based on the model name
+        if self.model_name.startswith("samba/"):
+            samba_api_key = os.environ.get("SAMBANOVA_API_KEY")
+            samba_base_url = os.environ.get("SAMBANOVA_BASE_URL", "https://api.sambanova.ai/v1") # Default if not set
+            if not samba_api_key:
+                raise ValueError("SAMBANOVA_API_KEY environment variable not set for SambaNova model.")
+            # For SambaNova (OpenAI compatible), explicitly pass api_key and api_base
+            # The model name for litellm should be just the model identifier, not the full "samba/" prefix if api_base is provided.
+            # However, litellm docs suggest that for OpenAI compatible endpoints, the model name passed to `completion`
+            # should be what the endpoint expects. The `model` param in `partial` here is the one sent in the request body.
+            # The `custom_llm_provider` in litellm is another way, but direct params are simpler for OpenAI compatibility.
+            # Let's try keeping the model name as is (e.g. "samba/DeepSeek-R1-Distill-Llama-70B")
+            # and provide api_key and api_base. LiteLLM should use these for any model if provided.
+            # If this doesn't work, the model name might need to be stripped of "samba/" if api_base is set.
+            # According to LiteLLM docs, for custom OpenAI-compatible endpoints, you can pass `base_url` and `api_key`.
+            # The `model` parameter to `litellm.completion` will be the actual model ID the endpoint expects.
+            # The `self.model_name` here is e.g. "samba/DeepSeek-R1-Distill-Llama-70B".
+            # We need to ensure the `model` argument to `completion` is just "DeepSeek-R1-Distill-Llama-70B"
+            # and set `custom_llm_provider="openai"` along with `api_base` and `api_key`.
+            # Or, if SambaNova is a recognized provider by a different name in litellm, use that.
+            # Given the error, litellm is not recognizing "samba/" as a provider directly.
+            # The simp            actual_model_name = self.model_name.split("samba/", 1)[1] if "samba/" in self.model_name else self.model_name
+            self.client = partial(completion,
+                                  model=actual_model_name,
+                                  api_key=samba_api_key,
+                                  api_base=samba_base_url,
+                                  custom_llm_provider="openai"
+                                 )
+            print(f"Initialized SambaNova model '{actual_model_name}' via custom OpenAI provider settings with api_base: {samba_base_url}")
+        elif self.model_name in known_model_list:
+            self.client = partial(completion, model=self.model_name)
+        else:
+            print(f"Warning: Model '{self.model_name}' not in explicit list and does not start with recognized prefixes. Attempting to initialize with litellm directly.")
+            try:
+                self.client = partial(completion, model=self.model_name)
+                print(f"Successfully initialized model '{self.model_name}' via litellm fallback.")
+            except Exception as e:                raise ValueError(f"Model '{self.model_name}' is not in the known list, does not start with recognized prefixes, and could not be initialized by litellm directly: {{e}}")
+        self.gpt4Tokenizer = tiktoken.encoding_for_model("gpt-4o")    
+
+    def count_tokens(self, text: str) -> int:
+        """
+        Count the number of tokens in the text.
+        """
+        tokens = self.gpt4Tokenizer.encode(text)
+        return len(tokens)
+
+    def generate(self,
+        history: List[str],
+        temperature: float = 0.1,
+        max_tokens: int = 2048,
+        current_depth: int = 1,
+        max_depth_num_rounds: int = 3,
+        allow_code_execution: bool = True,
+        code_execution_flag: str = "EXECUTE CODE!",
+        final_output: str = ""
+    ) -> str:
+        """
+        Generate a response from the language model.
+        """
+        if len(history) == 0:
+            raise ValueError("History must contain at least one message.")
+        
+        # The self.client is already a partial function with model, api_key, base_url, etc., pre-filled for SambaNova
+        response = self.client(
+            messages=history,
+            # model=self.model_name, # This is now part of the partial self.client for SambaNova
+            temperature=temperature,
+            max_tokens=max_tokens, # litellm uses max_tokens or max_completion_tokens
+        )
+        output = response.choices[0].message.content # Corrected access to content
+
+        pre_code_execution_flag = output.split(code_execution_flag)[0].strip()
+        if allow_code_execution and code_execution_flag in output and pre_code_execution_flag.endswith("```"):
+            output_prefix = output.split(code_execution_flag)[0].strip()
+            executed_code = extract_and_run_python_code(output_prefix)
+            executed_code = executed_code.strip()
+            current_output = f"{output_prefix}\n{code_execution_flag}\n\n{executed_code}"
+            final_output = f"{final_output}\n\n{current_output}".strip()
+
+            if current_depth <= max_depth_num_rounds:
+                warning_txt = ""
+                if current_depth == max_depth_num_rounds:
+                    warning_txt = f" (This is the last round. No more code execution will be allowed. Please present your final solution now.)"
+                new_messages = [
+                    {"role": "assistant", "content": current_output},
+                    {"role": "user", "content": f"Proceed with any additional steps required and provide the completed solution. If everything is already complete, type FINAL ANSWER and submit it in the expected format. If you are stuck, please try alternative methods to solve the problem and provide the final solution.{warning_txt}"}
+                ]
+                history += new_messages
+                return self.generate(
+                    history=history,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    current_depth=current_depth+1,
+                    max_depth_num_rounds=max_depth_num_rounds,
+                    allow_code_execution=allow_code_execution,
+                    code_execution_flag=code_execution_flag,
+                    final_output=final_output,
+                )
+            else:
+                return final_output
+        else:
+            final_output = f"{final_output}\n\n{output}".strip()
+            return final_output
+
+    def advanced_generate(self,
+        approach_name: str,
+        input_txt: str,
+        cheatsheet: str = None,
+        generator_template: str = None,
+        cheatsheet_template: str = None,
+        temperature: float = 0.0,
+        max_tokens: int = 2048,
+        max_num_rounds: int = 1,
+        allow_code_execution: bool = True,
+        code_execution_flag: str = "EXECUTE CODE!",
+        add_previous_answers_to_cheatsheet: bool = True,
+        original_input_corpus: List[str] = None,
+        original_input_embeddings: np.ndarray = None,
+        generator_outputs_so_far: List[str] = None,
+        retrieve_top_k: int = 3,
+    ) -> dict:
+        """
+        Generate a response from the language model.
+        Returns dict instead of Tuple for clarity.
+        """
+
+        if approach_name == "default":
+            generator_prompt = generator_template.replace("[[QUESTION]]", input_txt).replace("[[CHEATSHEET]]", "(empty)")
+            generator_history = [
+                {"role": "user", "content": generator_prompt},
+            ]
+            generator_output = self.generate(
+                history=generator_history,
+                temperature=temperature,
+                max_tokens=max_tokens,
+                allow_code_execution=allow_code_execution,
+                code_execution_flag=code_execution_flag,
+            )
+            generator_answer = extract_answer(generator_output)
+            return {
+                "input_txt": input_txt,
+                "steps": [
+                    {
+                        "round": 0,
+                        "generator_prompt": generator_prompt,
+                        "generator_output": generator_output,
+                        "generator_answer": generator_answer,
+                        "current_cheatsheet": None,
+                        "new_cheatsheet": None,
+                    }
+                ],
+                "previous_answers": None,
+                "final_answer": generator_answer,
+                "final_output": generator_output,
+                "final_cheatsheet": None,
+            }
+        
+        elif approach_name == "DynamicCheatsheet_Cumulative":
+            if cheatsheet is None:
+                raise ValueError("Cheatsheet must be provided for DynamicCheatsheet_Cumulative approach.")
+            if generator_template is None or cheatsheet_template is None:
+                raise ValueError("Generator and Cheatsheet templates must be provided for DynamicCheatsheet_Cumulative approach.")
+            
+            steps = []
+            previous_answers = []
+            current_cheatsheet_in_round = cheatsheet # Use a local var for the loop
+
+            for round_num in range(max(1, max_num_rounds)):
+                generator_cheatsheet_content = current_cheatsheet_in_round
+                if round_num > 0 and add_previous_answers_to_cheatsheet and previous_answers:
+                    previous_answers_txt = f"PREVIOUS ANSWERS:\n{'; '.join(previous_answers)}"
+                    generator_cheatsheet_content = f"{generator_cheatsheet_content}\n\n{previous_answers_txt}"
+
+                generator_prompt = generator_template.replace("[[QUESTION]]", input_txt).replace("[[CHEATSHEET]]", generator_cheatsheet_content)
+                
+                generator_history = [{"role": "user", "content": generator_prompt}]
+                generator_output = self.generate(
+                    history=generator_history,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    allow_code_execution=allow_code_execution,
+                    code_execution_flag=code_execution_flag,
+                )
+                generator_answer = extract_answer(generator_output)
+
+                cheatsheet_prompt = cheatsheet_template.replace("[[QUESTION]]", input_txt).replace("[[MODEL_ANSWER]]", generator_output).replace("[[PREVIOUS_CHEATSHEET]]", current_cheatsheet_in_round)
+                cheatsheet_history = [{"role": "user", "content": cheatsheet_prompt}]
+                # Pass explicit provider details for curator model if it's also SambaNova
+                # Assuming curator uses the same model instance for now.
+                cheatsheet_model_output = self.generate(
+                    history=cheatsheet_history,
+                    temperature=temperature,
+                    max_tokens=2*max_tokens, # As per original
+                    allow_code_execution=False,
+                )
+                new_cheatsheet = extract_cheatsheet(response=cheatsheet_model_output, old_cheatsheet=current_cheatsheet_in_round)
+                
+                steps.append({
+                    "round": round_num,
+                    "generator_prompt": generator_prompt,
+                    "generator_output": generator_output,
+                    "generator_answer": generator_answer,
+                    "current_cheatsheet": current_cheatsheet_in_round,
+                    "new_cheatsheet": new_cheatsheet,
+                })
+                current_cheatsheet_in_round = new_cheatsheet # Update for next potential round
+                if generator_answer:
+                    previous_answers.append(f"Round {round_num+1}: {generator_answer}")
+
+            return {
+                "input_txt": input_txt,
+                "steps": steps,
+                "previous_answers": previous_answers,
+                "final_answer": generator_answer, # Answer from the last round
+                "final_cheatsheet": current_cheatsheet_in_round, # Cheatsheet from the last round
+                "final_output": generator_output, # Full output from the last generator call
+            }
+        elif approach_name == "FullHistoryAppending":
+            length_of_history = len(generator_outputs_so_far) if generator_outputs_so_far else 0
+            curated_cheatsheet = "(empty)"
+            if length_of_history > 0 and original_input_corpus and generator_outputs_so_far:
+                curated_cheatsheet = "### PREVIOUS SOLUTIONS (START)\n\n"
+                for i, (prev_input, prev_output) in enumerate(zip(original_input_corpus[:length_of_history], generator_outputs_so_far[:length_of_history])):
+                    curated_cheatsheet += f"#### Previous Input #{i+1}:\n\n{prev_input}\n\n#### Model Solution to Previous Input #{i+1}:\n\n{prev_output}\n---\n---\n\n"
+                curated_cheatsheet += "#### PREVIOUS SOLUTIONS (END)"
+            
+            generator_prompt = generator_template.replace("[[QUESTION]]", input_txt).replace("[[CHEATSHEET]]", curated_cheatsheet)
+            generator_history = [{"role": "user", "content": generator_prompt}]
+            generator_output = self.generate(
+                    history=generator_history,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    allow_code_execution=allow_code_execution,
+                    code_execution_flag=code_execution_flag,
+                )
+            generator_answer = extract_answer(generator_output)
+            return {
+                "input_txt": input_txt,
+                "steps": [], 
+                "previous_answers": [],
+                "final_answer": generator_answer,
+                "final_cheatsheet": curated_cheatsheet, 
+                "final_output": generator_output,
+            }
+        else:
+            raise ValueError(f"Unknown approach_name: {approach_name}")
+

dynamic_cheatsheet/utils/evaluation.py

@@ -0,0 +1,293 @@
+import sys
+import re
+import os
+from typing import List
+# from .sonnet_eval import sonnet_errors
+from .execute_code import execute_code_with_timeout
+
+
+def clean_output_for_arithmetic(output: str) -> str:
+    """
+    Clean the output for arithmetic problems.
+
+    Args:
+        output (str): The output to clean.
+
+    Returns:
+        str: The cleaned output.
+    """
+    if "=" in output:
+        output = output.split("=")[1].strip()
+    if " is" in output:
+        output = output.split(" is")[1].strip()
+    if " equals" in output:
+        output = output.split(" equals")[1].strip()
+    if " evaluates to" in output:
+        output = output.split(" evaluates to")[1].strip()
+    if " is equal to" in output:
+        output = output.split(" is equal to")[1].strip()
+    return output
+
+
+def clean_output_for_GameOf24(output: str) -> str:
+    """
+    Clean the output for GameOf24 problems.
+    """
+    if "=" in output:
+        output = output.split("=")[0].strip()
+    if "is" in output:
+        output = output.split("is")[1].strip()
+    if "equals" in output:
+        output = output.split("equals")[0].strip()
+    if "evaluates to" in output:
+        output = output.split("evaluates to")[0].strip()
+    return output
+
+
+def eval_for_GameOf24(input: str, output: str) -> bool:
+    """
+    Given an input and output, check if the output is correct and follows the rules of the game.
+    """
+    clean_output = output
+
+    clean_output = clean_output_for_GameOf24(output)
+    clean_output = clean_output.replace("x", "*").strip()
+    clean_output = clean_output.replace("×", "*").strip()
+    clean_output = clean_output.replace("÷", "/").strip()
+    
+    try:
+        # Get the value of the expression using eval
+        value = eval(clean_output)
+        if not (abs(value - 24) < 1e-3):
+            return False
+        # Split the input and output digits by space
+        input_digits = input.split(" ")
+        # Replace the following symbols with space
+        replacements = ["+", "-", "*", "/", "÷", "(", ")"]
+        for symbol in replacements:
+            clean_output = clean_output.replace(symbol, " ")
+        # Replace multiple spaces with single space
+        clean_output = re.sub(" +", " ", clean_output)
+        clean_output = clean_output.strip()
+        output_digits = clean_output.split(" ")
+        # Sort the digits
+        input_digits.sort()
+        output_digits.sort()
+        # Check if the digits are the same
+        if input_digits != output_digits:
+            return False
+        return True
+    except Exception as e:
+        return False
+
+
+def remove_punctuation(output: str) -> str:
+    """
+    Remove punctuation from the output.
+    """
+    markers = [",", ";", ":", ".", '"']
+    for marker in markers:
+        output = output.replace(marker, "")
+    return output
+
+
+def convert_newline_to_space(output: str) -> str:
+    """
+    Convert newline to space.
+    """
+    output = output.replace("\n", " ")
+    return output
+
+
+def eval_for_exact_matching_with_no_punctuation(
+    output: str, target: str
+) -> bool:
+    """
+    Evaluate if the output is exactly the same as the target.
+    """
+    output = remove_punctuation(output)
+    output = convert_newline_to_space(output)
+    if target == output:
+        return True
+    return False
+
+
+def eval_for_softmatch(input: str, output: str, target: str) -> bool:
+    """
+    Evaluate if the output is a soft match of the target.
+    """
+    output = remove_punctuation(output)
+    if target in output:
+        return True
+    return False
+
+
+def eval_for_CheckmateInOne(input: str, output: str, target: str) -> bool:
+    """
+    Evaluate if the output is a checkmate in one.
+    """
+    output = output.strip()
+    if output[-1] == "#":
+        output = output.split(" ")[-1].strip()
+    # Based on the input, determine the number of the last move
+    last_move = input.split(".")[-1].strip()
+    move_idx = input.split(".")[-2].split(" ")[-1].strip()
+    # If the last move is an empty string, then the last move is white; otherwise, it is black
+    if last_move == "":
+        last_move = "White"
+    else:
+        last_move = "Black"
+    next_move_idx = str(int(move_idx) + 1)
+    if not (next_move_idx in output):
+        if target in output or (target[1] == 'x' and (target[0] + target[2:]) in output):
+            return True
+    else:
+        output = output.split(next_move_idx)[0].strip()
+        if target in output or (target[1] == 'x' and (target[0] + target[2:]) in output):
+            return True
+    return False
+
+
+def eval_equation_balancer(input: str, output: str, target: str) -> bool:
+    """
+    Evaluate if the output is a valid equation balancer.
+    """
+    output = output.split("=")[0].strip()
+    target_val = target.split("=")[1].strip()
+    target = target.split("=")[0].strip()
+    # First make sure that the output has the same format as the target (when operators (e.g., +, -, *, /) are removed)
+    output_nums = output.replace("+", "").replace("-", "").replace("*", "").replace("/", "").replace(" ", "").strip()
+    target_nums = target.replace("+", "").replace("-", "").replace("*", "").replace("/", "").replace(" ", "").strip()
+    if output_nums != target_nums:
+        return False
+    # Now, evaluate the output and target
+    try:
+        output_value = eval(output)
+        if abs(output_value - eval(target_val)) < 1e-6:
+            return True
+    except Exception as e:
+        return False
+    return False
+
+
+def eval_for_multiple_choice(input_text: str, final_answer: str, target: str) -> bool:
+    """
+    Evaluates if the final answer matches the target using pattern matching.
+    
+    Args:
+        input_text (str): The original question text including options
+        final_answer (str): The model's answer
+        target (str): The correct answer
+    
+    Returns:
+        bool: True if answer is correct, False otherwise
+    """
+    # Handle empty or None inputs
+    if not final_answer or not target:
+        return False
+    
+    def clean_text(text: str) -> str:
+        if not text:
+            return ""
+        return text.lower().strip().replace('`', '').replace('(', '').replace(')', '')
+    
+    def extract_option_text(input_text: str, option_letter: str) -> str:
+        try:
+            # Try different formats of options sections
+            options_section = ""
+            if 'options:' in input_text.lower():
+                options_section = input_text.lower().split('options:')[1].strip()
+            elif 'choices:' in input_text.lower():
+                options_section = input_text.lower().split('choices:')[1].strip()
+            
+            if not options_section:
+                # Try to find options in the format (A) text, (B) text
+                lines = input_text.lower().split('\n')
+                for i, line in enumerate(lines):
+                    if line.strip().startswith(f'({option_letter})') or line.strip().startswith(f'{option_letter})'):
+                        return line.split(')', 1)[1].strip()
+                
+            # Process the options section if found
+            for line in options_section.split('\n'):
+                line = line.strip()
+                if line.startswith(f'({option_letter})') or line.startswith(f'{option_letter})'):
+                    return line.split(')', 1)[1].strip()
+                # Handle options like "A. text" format
+                if line.startswith(f'{option_letter}.'):
+                    return line.split('.', 1)[1].strip()
+        except:
+            return ''
+        return ''
+
+    # Full option match (A), (B), etc. (e.g., (A) == (A))
+    if final_answer == target:
+        return True
+
+    # Clean and normalize inputs
+    clean_answer = clean_text(final_answer)
+    clean_target = clean_text(target)
+    
+    # Handle target formats: (A), A), A, etc.
+    target_letter = ""
+    if len(clean_target) == 1:
+        target_letter = clean_target
+    elif clean_target.endswith(')'):
+        target_letter = clean_target[-2]
+    else:
+        # Extract the last character if it's a letter a-d or A-D
+        last_char = clean_target[-1]
+        if last_char in 'abcd':
+            target_letter = last_char
+    
+    # Direct letter match (a, b, c, d)
+    if len(clean_answer) == 1 and clean_answer in 'abcd' and clean_answer == target_letter:
+        return True
+    
+    # Handle answer formats like "A" or "A."
+    if clean_answer.startswith(target_letter) and (len(clean_answer) == 1 or 
+                                                  (len(clean_answer) == 2 and clean_answer[1] == '.')):
+        return True
+    
+    # Handle answer formats like "Option A" or "Answer is A"
+    if clean_answer.endswith(target_letter) and (clean_answer[-2:] == f" {target_letter}" or 
+                                               clean_answer[-3:] == f" {target_letter}."):
+        return True
+    
+    # Text content match - check if the target option text is in the answer
+    target_text = extract_option_text(input_text, target_letter)
+    
+    if target_text and target_text in clean_answer:
+        return True
+    
+    # Handle numerical answers (if target is a number and answer contains that number)
+    if target_letter.isdigit() and target_letter in clean_answer:
+        return True
+        
+    return False
+
+
+def eval_for_pyton_programming_puzzles(input: str, output: str) -> bool:
+    """
+    Evaluate if the output is a valid Python programming puzzle solution.
+    """
+    if "```python" in output:
+        output = output.split("```python")[-1].strip()
+        output = output.split("```")[0].strip()
+
+    if "def sat" in output:
+        if "from typing" not in output:
+            output = f"from typing import *\n{output}"
+        code = f"{output}\nanswer = solution()\nprint(sat(answer))"
+    else:
+        code = f"from typing import *\n{input}\n{output}\nanswer = solution()\nprint(sat(answer))"
+    
+    code = code.replace("List[", "list[")
+    eval_bool = execute_code_with_timeout(code, timeout=3)
+
+    if "NameError: name 'answer' is not defined" in eval_bool:
+        print(f"Eval bool: {eval_bool}")
+        print(f"Code:\n{code}")
+        print("*" * 100)
+    if "True" in eval_bool:
+        return True
+    return False

dynamic_cheatsheet/utils/execute_code.py

@@ -0,0 +1,101 @@
+"""
+This module provides functions to extract and execute Python code from a string.
+
+The functions are:
+* extract_and_run_python_code(txt: str) -> str: Extracts and executes Python code from a string.
+* execute_code_with_timeout(code: str, timeout: int = 3) -> str: Executes Python code with a timeout and returns the output.
+
+Additional functions can be added as needed.
+"""
+
+import os
+import tempfile
+from subprocess import Popen, PIPE, TimeoutExpired
+
+def extract_and_run_python_code(txt: str) -> str:
+    """
+    Extract and execute Python code from a provided string.
+
+    Handles missing print statements for non-comment last lines,
+    executes the code, and captures output or errors.
+
+    Parameters:
+        txt (str): Input string containing a possible Python code block.
+
+    Returns:
+        str: Execution result or error message wrapped in output formatting.
+    """
+    def extract_code(input_str: str) -> str:
+        """Extract Python code block delimited by ```python and ```."""
+        try:
+            return input_str.split("```python", 1)[1].split("```", 1)[0].strip()
+        except IndexError:
+            raise ValueError("No valid Python code block found.")
+
+    def ensure_print_statement(code: str) -> str:
+        """
+        Append a print statement if the last line isn't a comment or a print statement.
+        """
+        lines = code.splitlines()
+        last_line = lines[-1].rstrip()
+        if not last_line.startswith(("print(", "#", " ", "\t")) and (not ("return" in last_line)):# and len((last_line.split(" "))) == 1:
+            lines[-1] = f"print({last_line})"
+        return "\n".join(lines)
+
+    if "```python" not in txt:
+        return None  # Return early if no Python code block is present
+
+    try:
+        # Extract and sanitize the code
+        code_block = extract_code(txt)
+        code_with_print = ensure_print_statement(code_block)
+
+        # Execute the code and return output
+        python_output = execute_code_with_timeout(code_with_print)
+        # return f"PYTHON CODE OUTPUT:\n'''\n{python_output}\n'''"
+        return f"Output of the Python code above:\n```\n{python_output}\n```"
+
+    except Exception as error:
+        return f"PYTHON CODE OUTPUT:\n```\nError: {str(error)}\n```"
+
+
+# Python code execution function with timeout
+# TODO (msuzgun): Improve the security of this function by using a sandboxed environment
+def execute_code_with_timeout(code: str, timeout: int = 3) -> str:
+    """
+    Execute Python code with a timeout and return the output.
+    
+    Parameters:
+        code (str): Python code to execute.
+        timeout (int): Timeout duration in seconds.
+
+    Returns:
+        str: Captured output or error message from the code execution.
+    """
+    with tempfile.NamedTemporaryFile(
+        mode="w+t", suffix=".py", delete=False
+    ) as temp_file:
+        temp_file.write(code)
+        temp_file.flush()
+
+    try:
+        # In case alias python=python3 is not set, use python3 instead of python
+        process = Popen(["python3", temp_file.name], stdout=PIPE, stderr=PIPE)
+        stdout, stderr = process.communicate(timeout=timeout)
+        captured_output = stdout.decode().strip()
+        error_output = stderr.decode().strip()
+
+        if captured_output == "":
+            if error_output != "":
+                captured_output = f"Error in execution: {error_output}"
+            else:
+                captured_output = "(No output was generated. It is possible that you did not include a print statement in your code. If you want to see the output, please include a print statement.)"
+
+    except TimeoutExpired:
+        process.kill()
+        captured_output = "Execution took too long, aborting..."
+
+    finally:
+        os.remove(temp_file.name)
+
+    return captured_output

dynamic_cheatsheet/utils/extractor.py

@@ -0,0 +1,115 @@
+"""
+This file contains the functions to extract the final answer, cheatsheet and solution evaluation from model responses.
+
+The functions are:
+* extract_answer(response: str) -> str: Extracts the final answer from the model response.
+* extract_cheatsheet(response: str, old_cheatsheet: str) -> str: Extracts the cheatsheet from the model response.
+* extract_solution(response: str, header: str = "SOLUTION EVALUATION:", error_message : str = "No solution evaluation found") -> str: Extracts the solution evaluation from the model response.
+
+Additional functions can be added as needed.
+"""
+
+def extract_answer(
+    response: str,
+) -> str:
+    """
+    Extracts the final answer from the model response.
+
+    Arguments:
+        response : str : The response from the model.
+
+    Returns:
+        str : The extracted final answer (if not found, returns "No final answer found").
+    """
+    if "<answer>" in response:
+        # <answer> (content) </answer>
+        try:
+            txt = response.split("<answer>")[-1].strip()
+            txt = txt.split("</answer>")[0].strip()
+            return txt
+        except:
+            return "No final answer found"
+    else:
+        if not("FINAL ANSWER" in response):
+            return "No final answer found"
+        try:
+            response = response.split("FINAL ANSWER")[-1].strip()
+            if response[0] == ":":
+                response = response[1:].strip()
+
+            # First decide whether to split by "```" or "'''" based on the presence of "```" or "'''"
+            idx_1 = response.find("'''")
+            idx_2 = response.find("```")
+            if min(idx_1, idx_2) != -1: 
+                if idx_1 < idx_2:
+                    response = response.split("'''")[1].strip()
+                else:
+                    response = response.split("```")[1].strip()
+            else:
+                if idx_1 == -1:
+                    response = response.split("```")[1].strip()
+                else:
+                    response = response.split("'''")[1].strip()
+
+            # Special case for P3-Test task: If the first line contains "python" then remove it
+            if response.split("\n")[0].strip().lower() == "python":
+                response = "\n".join(response.split("\n")[1:]).strip()
+            return response
+        except:
+            return "No final answer found"
+
+
+def extract_cheatsheet(
+    response: str,
+    old_cheatsheet: str,
+) -> str:
+    """
+    Extracts the cheatsheet from the model response.
+    
+    Arguments:
+        response : str : The response from the model.
+        old_cheatsheet : str : The old cheatsheet to return if the new one is not found.
+
+    Returns:
+        str : The extracted cheatsheet (if not found, returns the old cheatsheet).
+    """
+    response = response.strip()
+    # <cheatsheet> (content) </cheatsheet>
+    if "<cheatsheet>" in response:
+        try:
+            txt = response.split("<cheatsheet>")[1].strip()
+            txt = txt.split("</cheatsheet>")[0].strip()
+            return txt
+        except:
+            return old_cheatsheet
+    else:
+        return old_cheatsheet
+
+
+def extract_solution(
+    response: str,
+    header: str = "SOLUTION EVALUATION:",
+    error_message : str = "No solution evaluation found",
+) -> str:
+    """
+    Extracts the solution evaluation from the model response.
+
+    Arguments:
+        response : str : The response from the model.
+        header : str : The header to search for the solution evaluation.
+        error_message : str : The error message to return if the solution evaluation is not found.
+
+    Returns:
+        str : The extracted solution evaluation (if not found, returns the error message).
+    """
+    response = response.strip()
+    try:
+        txt = response.split(header)[1]
+        try:
+            txt = txt.split("'''")[1].strip()
+        except:
+            return txt.strip()
+    except:
+        return response
+        # return error_message
+    return txt

dynamic_cheatsheet/utils/sonnet_eval.py

@@ -0,0 +1,511 @@
+"""
+Defines function sonnet_errors(poem, target: str) -> Dict[str, Any]
+which takes a target rhyme scheme (and optionally a list of required words) and returns a dict of errors
+
+Returns an empty dictionary if there are no errors, so bool(sonnet_errors(poem, target)) is False if there are no
+errors. It's a permissive check for sonnets errors, meaning that if it is unsure then it doesn't return an error.
+
+Specifically, 
+
+* Check if it adheres to a given rhyming scheme
+* Check if each line has 10-11 syllables, more precisely, there's some pronounciation of each line with 10-11 syllalbes
+
+This omits a few things like rhymes and iambic pentameter.
+
+# Rhymes
+
+For rhymes, we use python `pronouncing` library based on:
+
+* CMU pronouncing dictionary http://www.speech.cs.cmu.edu/cgi-bin/cmudict
+
+# Syllable counting 
+
+Given that there are multiple ways to pronounce many words (e.g. "caramel" can be pronounced with 2 or 3 syllables),
+we adopt a "permissive" approach and consult multiple tools for syllable counting:
+
+* pronounce - a well-known pronunciation dict based on from CMU's pronouncing dictionary 
+* syllables - a Python library for syllable counting
+* pyphen - a Python wrapper for the hyphenation library
+"""
+
+from typing import Set, Dict, Any
+import re
+import joblib
+import pyphen
+import syllables
+import pronouncing
+
+
+ALLOWED_SYLLABLES = {
+    10,
+    11,
+}  # about 3-4% of legit lines have 11 syllables, so we allow it, > 99% have 10 or 11
+NUM_REQUIRED_WORDS = 3
+
+memory = joblib.Memory(
+    ".cache", verbose=0
+)  # use cache to speed up repeated rhyme/syllable calls
+
+
+def sonnet_errors(poem: str, target: str, verbose=False) -> Dict[str, Any]:
+    """
+    Checks for sonnet errors with respect to target rhyme scheme (and optional required words)
+
+    args:
+        poem: the poem to check
+        target: the rhyme scheme, e.g. "ABBA ABBA CDC DCD"
+                optionally target can have a list of required words, like
+                "ABBA ABBA CDC DCD, love train snail" each of these must be in the poem
+        verbose: if True, print out more details
+    """
+    if ", " in target:
+        scheme, rest = target.split(", ")
+        required_words = rest.split()
+    else:
+        scheme = target
+        required_words = []
+
+    errors = scheme_errors(poem, scheme, verbose=verbose)
+    assert isinstance(errors, dict)
+    missing_words = [w for w in required_words if w.lower() not in poem.lower()]
+    if any(missing_words):
+        errors["missing words"] = missing_words
+
+    syllable_errors = []
+    for line in split_poem(poem):
+        variations = syllable_variations(line)
+        if not (variations & ALLOWED_SYLLABLES):
+            syllable_errors.append((line, sorted(variations)))
+    if syllable_errors:
+        errors["syllable errors"] = syllable_errors
+
+    return errors
+
+
+def clean_word(text: str):
+    return text.lower().strip(",.!?;: \"'[]()/")
+
+
+def clean_line(line: str):
+    """
+    Clean a line from a poem.
+    Check if line ends with (A) or (B) ... and remove it
+    """
+    line = re.sub(r"\s*\([A-Za-z]\)\s*$", "", line)
+    return line.strip()
+
+
+def split_poem(poem: str, min_line_len=3):
+    ans = [clean_line(l) for l in poem.splitlines()]
+    return [l for l in ans if len(l) > min_line_len]
+
+
+@memory.cache
+def slant_rhyming_parts(word: str):
+    consonants = set("BCDFGHJKLMNPQRSTVWXYZ")
+    ans = [
+        "".join(
+            ("R" if "R" in p else (p if p in consonants else "?"))
+            for p in pronouncing.rhyming_part(ph).split()
+        )
+        for ph in pronouncing.phones_for_word(word)
+    ]
+    ans = [a for a in ans if not all(i == "?" for i in a)]
+    ans = [a.replace("?", "") + ("?" if a.endswith("?") else "") for a in ans]
+    return set(ans)
+
+
+@memory.cache
+def get_rhymes(w):
+    return set(pronouncing.rhymes(w))
+
+
+def scheme_errors(poem: str, scheme: str, verbose=False):
+    """Find errors with respect to a given rhyming scheme"""
+    lines = split_poem(poem)
+    scheme = scheme.replace(" ", "")
+
+    if len(lines) != len(scheme):
+        return {
+            "line count": f"Poem has {len(lines)} != {len(scheme)} lines in pattern {scheme}"
+        }
+
+    last_words = [clean_word(l.replace("-", " ").split()[-1]) for l in lines]
+
+    dictionary = pronouncing.cmudict.dict()  # we ignore words not in dictionary
+
+    groups = []
+    for chars in sorted(set(scheme)):
+        groups.append(
+            [w for w, p in zip(last_words, scheme) if p == chars and w in dictionary]
+        )
+
+    slant_sets = {w: set(slant_rhyming_parts(w)) for g in groups for w in g}
+
+    scores = {}
+
+    if verbose:
+        print(groups)
+
+    for g in groups:
+        internal_words = set(g)
+        external_words = {w for h in groups if h is not g for w in h}
+        if len(internal_words) == 1:
+            continue  # don't check rhymes if only word word in the group is in dictionary
+        for w in g:
+            rhymes = get_rhymes(w)
+            scores[w] = []
+            for comparisons in [internal_words, external_words]:
+                m = dict(rhymes=[], slant_rhymes=[])
+                scores[w].append(m)
+                for v in comparisons:
+                    if v == w:
+                        continue
+                    if v in rhymes:
+                        m["rhymes"].append(v)
+                    elif slant_sets[v] & slant_sets[w]:
+                        m["slant_rhymes"].append(v)
+
+    error_reasons = {}
+    suspicious_reasons = {}
+
+    for w in scores:
+        internal, external = scores[w]
+
+        if internal["rhymes"] or internal["slant_rhymes"]:
+            pass  # ok if it rhymes (perfect or slant) with at least one other word in the group
+        elif len(external["rhymes"]) >= 2:
+            error_reasons[w] = "no internal rhymes, 2+ external perfect rhymes"
+        elif external["rhymes"]:
+            if len(external["slant_rhymes"]) >= 2:
+                error_reasons[
+                    w
+                ] = "no internal rhymes, 1 external perfect rhyme, 2+ external slant rhymes"
+            else:
+                suspicious_reasons[
+                    w
+                ] = "no internal rhymes/slant rhymes, 1 external perfect rhymes"
+        elif len(external["slant_rhymes"]) >= 3:
+            error_reasons[
+                w
+            ] = "no internal rhymes/slant rhymes, 3+ external slant rhymes"
+        if verbose:
+            print(w, "internal:", internal, "external:", external)
+
+    if len(error_reasons) + len(suspicious_reasons) >= 3:
+        error_reasons.update(suspicious_reasons)
+
+    return {
+        w: {
+            "reason": error_reasons[w],
+            "internal": scores[w][0],
+            "external": scores[w][1],
+        }
+        for w in error_reasons
+    }
+
+
+def syllable_variations(text, verbose=False) -> Set[int]:
+    """
+    Given a text, return the set of possible numbers of syllables. It's a set because some words like "caramel" can
+    be pronounced with different numbers of syllables.
+    """
+    ans = {0}
+    for word in re.split("[ -]+", text):
+        word = clean_word(word)
+        if not word:
+            continue
+        options = word_syllables(word)
+        options = range(
+            min(options), max(options) + 1
+        )  # make it a range (so {2, 4} moves to [2, 3, 4])
+        ans = {x + y for x in ans for y in options}
+    return ans
+
+
+@memory.cache
+def word_syllables(word: str) -> Set[int]:
+    assert word == clean_word(
+        word
+    ), "Word should be cleaned before hitting word_syllables cache"
+    return SyllableCounters.count_word(word)
+
+
+class SyllableCounters:
+    """
+    Simple class to count syllables in text.
+    """
+
+    _cmu_dict = None
+    _pyphen_counter = None
+
+    @staticmethod
+    def cmu_dict():
+        if not SyllableCounters._cmu_dict:
+            SyllableCounters._cmu_dict = pronouncing.cmudict.dict()
+        return SyllableCounters._cmu_dict
+
+    def cmu(word):
+        return {
+            pronouncing.syllable_count(pro) for pro in pronouncing.phones_for_word(word)
+        }
+
+    @staticmethod
+    def pyphen_counter():
+        if not SyllableCounters._pyphen_counter:
+            SyllableCounters._pyphen_counter = pyphen.Pyphen(lang="en")
+        return SyllableCounters._pyphen_counter
+
+    @staticmethod
+    def count_word(word) -> Set[int]:
+        if not word:
+            return {0}
+
+        cmu = SyllableCounters.cmu(word)
+
+        pyph = SyllableCounters.pyphen_counter().inserted(word).count("-") + 1
+
+        syll = syllables.estimate(word)
+
+        ans = cmu | {pyph, syll}
+
+        if 0 in ans and len(ans) > 1:
+            ans.remove(0)
+
+        return ans
+
+
+TESTS = [
+    ["In savannah where tall trees kiss the sky,", 10],
+    ["A giraffe named Joe with love-stricken grace,", 10],
+    ["Did find a turtle named Sarah nearby,", 10],
+    ["Their eyes did meet, hearts raced in sweet embrace.", 10],
+    ["Though nature's laws deemed their love quite absurd,", 10],
+    ["Joe's neck would bend to whisper words of flame,", 10],
+    ["And Sarah's shell would tremble at each word,", 10],
+    ["In love's bizarre dance, they found no one to blame.", 11],
+    ["Through sun and storm, they'd wander, hoof and claw,", 10],
+    ["With love that no one ever could unravel,", 11],
+    ["In each other's eyes, perfection they saw,", 10],
+    ["A love so fierce, no distance could they travel.", 11],
+    ["So let us learn from turtle and giraffe,", 10],
+    ["That love's own shape can make the coldest laugh.", 10],
+    ["In yonder sky where colours blend so high,", 10],
+    ["A rainbow arcs, a bridge 'twixt earth and air.", 10],
+    ["Its radiant hues draw every gazing eye,", 12],
+    ["A painter's dream, a sight beyond compare.", 10],
+    ["Yet in the world of man, delight so small,", 10],
+    ["As gumball's sphere, with colours bright and clear.", 10],
+    ["Such simple joy it brings to one and all,", 10],
+    ["Its sweetness matched by colours we hold dear.", 10],
+    ["Both nature's arc and candy sphere delight,", 10],
+    ["The vast expanse and tiny bite unite,", 10],
+    ["In tales of wonder, stories to be told.", 10],
+    ["So let us cherish both the grand and small,", 10],
+    ["For beauty’s found in rainbow and in gumball.", 11],
+    ["When night's embrace hath shrouded all in black,", 10],
+    ["A flashlight's beam doth pierce the dark so deep,", 10],
+    ["From paths we've chosen, and vows we mean to keep.", 11],
+    ["Thou art like that beam, true, clear, and bright,", 9],
+    ["Cutting through the fog of my mind's own night,", 10],
+    ["Yet oft I find, by folly or by chance,", 10],
+    ["Distractions lead my wandering glance.", 9],
+    ["But even as stars, obscured by fleeting cloud,", 11],
+    ["Return to grace the heavens, proud and loud,", 10],
+    ["So shall my focus, once by ails distraught,", 10],
+    ["Return to thee, as ever it hath sought.", 10],
+    ["For in this world of fleeting sight and sound,", 10],
+]
+
+
+def fixed_tests():
+    failures = []
+    for line, expected in TESTS:
+        variations = syllable_variations(line)
+        if expected not in variations:
+            print(f"Line `{line}` has {expected} syllables which isn't in {variations}")
+            failures.append((line, expected, variations))
+
+    # tests from https://www.mentalfloss.com/article/53661/car-mel-or-car-mel-3-reasons-syllabically-ambiguous-words :
+    for words, expected in [
+        (
+            "fire tire hour liar buyer flower drawer layer loyal royal file orange poem crayon".split(),
+            [1, 2],
+        ),
+        (
+            "caramel mayonnaise family chocolate camera different separate favorite realtor".split(),
+            [2, 3],
+        ),
+        ("mischievous".split(), [3, 4]),
+    ]:
+        for w in words:
+            variations = syllable_variations(w)
+            for i in expected:
+                if i not in variations:
+                    print(
+                        f"{w} give syllable_variations {variations} but should include {i}"
+                    )
+                    failures.append((w, i, variations))
+    return failures
+
+
+def summarize_errors(errors, num_samples):
+    print(
+        f"Sonnet failure rate: {len(errors)/num_samples:.1%} out of {num_samples:,}, breakdown:"
+    )
+    wnl = sum("line count" in e for e in errors.values()) / num_samples
+    print(f"{wnl:.1%} wrong number of lines")
+    mw = sum(bool("missing words" in e) for e in errors.values()) / num_samples
+    print(f"{mw:.1%} missing words")
+    bl = sum(bool("syllable errors" in e) for e in errors.values()) / num_samples
+    print(f"{bl:.1%} poems with at least one line with wrong number of syllables")
+    rhyme_errors = (
+        sum(any(" " not in k for k in e) for e in errors.values()) / num_samples
+    )
+    both = (
+        sum(
+            (bool("syllable errors" in e) and any(" " not in k for k in e))
+            for e in errors.values()
+        )
+        / num_samples
+    )
+    print(
+        f"{rhyme_errors:.1%} poems with rhyme errors ({both:.1%} poems with both rhyme and syllable errors)"
+    )
+
+
+def corpus_check_scheme(corpus_filename, scheme):
+    with open(corpus_filename, "r") as f:
+        poems = [p.strip() for p in f.read().split("\n\n") if p]
+    errors = {}
+    for p in poems:
+        e = sonnet_errors(p, scheme)
+        if e:
+            errors[p] = e
+            print("*" * 50)
+            sonnet_errors(p, scheme, verbose=True)
+            print("scheme", scheme)
+            print(p)
+            print()
+            print(e)
+            print("<" * 50)
+
+    summarize_errors(errors, len(poems))
+
+
+def test():
+    assert not sonnet_errors(
+        """Not like the brazen giant of Greek fame,
+        With conquering limbs astride from land to land;
+        Here at our sea-washed, sunset gates shall stand
+        A mighty woman with a torch, whose flame
+        Is the imprisoned lightning, and her name
+        Mother of Exiles. From her beacon-hand
+        Glows world-wide welcome; her mild eyes command
+        The air-bridged harbor that twin cities frame.
+
+        "Keep, ancient lands, your storied pomp!" cries she
+        With silent lips. "Give me your tired, your poor,
+        Your huddled masses yearning to breathe free,
+        The wretched refuse of your teeming shore.
+        Send these, the homeless, tempest-tost to me,
+        I lift my lamp beside the golden door!"
+                """,
+        "ABBA ABBA CDCDCD",
+    )
+
+    assert not sonnet_errors(
+        """How do I love thee? Let me count the ways.
+        I love thee to the depth and breadth and height
+        My soul can reach, when feeling out of sight
+        For the ends of being and ideal grace.
+        I love thee to the level of every day’s
+        Most quiet need, by sun and candle-light.
+        I love thee freely, as men strive for right.
+        I love thee purely, as they turn from praise.
+        I love thee with the passion put to use
+        In my old griefs, and with my childhood’s faith.
+        I love thee with a love I seemed to lose
+        With my lost saints. I love thee with the breath,
+        Smiles, tears, of all my life; and, if God choose,
+        I shall but love thee better after death.""",
+        "abba abba cdcdcd",
+    )
+
+    assert not sonnet_errors(
+        """When, in disgrace with fortune and men’s eyes,
+        I all alone beweep my outcast state,
+        And trouble deaf heaven with my bootless cries,
+        And look upon myself, and curse my fate,
+        Wishing me like to one more rich in hope,
+        Featur’d like him, like him with friends possess’d,
+        Desiring this man’s art and that man’s scope,
+        With what I most enjoy contented least;
+        Yet in these thoughts myself almost despising,
+        Haply I think on thee, and then my state,
+        Like to the lark at break of day arising
+        From sullen earth, sings hymns at heaven’s gate;
+        For thy sweet love remember’d such wealth brings
+        That then I scorn to change my state with kings.""",
+        "ABAB CDCD EFEF GG",
+    )
+
+    assert sonnet_errors(
+        """How do I love thee? Let me count the ways.
+        I love thee to the depth and breadth and height
+        My soul can reach, when feeling out of sight
+        For the ends of being and ideal grace.
+        I love thee to the level of every day’s
+        Most quiet need, by sun and candle-light.
+        I love thee freely, as men strive for right.
+        I love thee purely, as they turn from praise.
+        I love thee with the passion put to use
+        In my old griefs, and with my childhood’s faith.
+        I love thee with a love I seemed to lose
+        With my lost saints. I love thee with the breath,
+        Smiles, tears, of all my life; and, if God choose,
+        I shall but love thee better after death.""",
+        "ABAB CDCD EFEF GG",
+    )
+
+    aaa = sonnet_errors(
+        """How do I love thee? Let me count the ways.
+        I love thee to the depth and breadth and height
+        My soul can reach, when feeling out of sight
+        For the ends of being and ideal grace.
+        I love thee to the level of every day’s
+        Most quiet need, by sun and candle-light.
+        I love thee freely, as men strive for right.
+        I love thee purely, as they turn from praise.
+        I love thee with the passion put to use
+        In my old griefs, and with my childhood’s faith.
+        I love thee with a love I seemed to lose
+        With my lost saints. I love thee with the breath,
+        Smiles, tears, of all my life; and, if God choose,
+        I shall but love thee better after death.""",
+        "ABBA ABBA CDC DCD",
+        # abba abba cdc dcd: (correct)
+        # "ABAB CDCD EFEF GG", (false)
+    )
+
+    print(aaa)
+
+    aaa = sonnet_errors(
+        """How do I love thee? Let me count the ways (A)
+        I love thee to the depth and breadth and height (B)
+        My soul can reach, when feeling out of sight (B)
+        For the ends of being and ideal grace (A)
+        I love thee to the level of every day’s (A)
+        Most quiet need, by sun and candle-light (B)
+        I love thee freely, as men strive for right (B)
+        I love thee purely, as they turn from praise (A)
+        I love thee with the passion put to use (C)
+        In my old griefs, and with my childhood’s faith (D)
+        I love thee with a love I seemed to lose (C)
+        With my lost saints. I love thee with the breath (D)
+        Smiles, tears, of all my life; and, if God choose (C)
+        I shall but love thee better after death (D).""",
+        "ABBA ABBA CDC DCD",
+        # abba abba cdc dcd: (correct)
+        # "ABAB CDCD EFEF GG", (false)
+    )

prompts/curator_prompt_for_dc_cumulative.txt

@@ -0,0 +1,149 @@
+# CHEATSHEET REFRENCE CURATOR
+
+#### 1. Purpose and Goals
+As the Cheatsheet Curator, you are tasked with creating a continuously evolving reference designed to help solve a wide variety of tasks, including algorithmic challenges, debugging, creative writing, and more. The cheatsheet's purpose is to consolidate verified solutions, reusable strategies, and critical insights into a single, well-structured resource.
+
+- The cheatsheet should include quick, accurate, reliable, and practical solutions to a range of technical and creative challenges. 
+- After seeing each input, you should improve the content of the cheatsheet, synthesizing lessons, insights, tricks, and errors learned from past problems and adapting to new challenges.
+
+---
+
+#### 2. Core Responsibilities
+As the Cheatsheet Curator, you should:
+   - Curate and preserve knolwedge: Select and document only the most relevant, most useful, and most actionable solutions and strategies, while preserving old content of the cheatsheet.
+   - Maintain accuracy: Ensure that all entries in the cheatsheet are accurate, clear, and well-contextualized. 
+   - Refine and update content: Continuously update and improve the content of the cheatsheet by incorporating new insights and solutions, removing repetitions or trivial information, and adding efficient solutions.
+   - Ensure practicality and comprehensiveness: Provide critical and informative examples, as well as efficient code snippets and actionable guidelines. 
+
+Before updating the cheatsheet, however, you should first assess the correctness of the provided solution and strategically incorporate code blocks, insights, and solutions into the new cheatsheet. Always aim to preserve and keep correct, useful, and illustrative solutions and strategies for future cheatsheets.
+
+---
+
+#### 3. Principles and Best Practices
+1. Accuracy and Relevance:
+   - Only include solutions and strategies that have been tested and proven effective.
+   - Clearly state any assumptions, limitations, or dependencies (e.g., specific Python libraries or solution hacks).
+   - For computational problems, encourage Python usage for more accurate calculations.
+
+2. Iterative Refinement:
+   - Continuously improve the cheatsheet by synthesizing both old and new solutions, refining explanations, and removing redundancies.
+   - Rather than deleting old content and writing new content each time, consider ways to maintain table content and synthesize information from multiple solutions.
+   - After solving a new problem, document any reusable codes, algorithms, strategies, edge cases, or optimization techniques. 
+
+3. Clarity and Usability:
+   - Write concise, actioanble, well-structured entries.
+   - Focus on key insights or strategies that make solutions correct and effective.
+
+4. Reusability:
+   - Provide clear solutions, pseudocodes, and meta strategies that are easily adaptable to different contexts.
+   - Avoid trivial content; focus on non-obvious, critical solution details and approaches.
+   - Make sure to add as many examples as you can in the cheatsheet. 
+   - Any useful, efficient, generalizable, and illustrative solutions to the previous problems should be included in the cheatsheet.
+
+---
+
+#### 4. Cheatsheet Structure
+The cheatsheet can be divided into the following sections:
+
+1. Solutions, Implementation Patterns, and Code Snippets:
+   - Document reusable code snippets, algorithms, and solution templates.
+   - Include descriptions, annotated examples, and potential pitfalls, albeit succinctly.
+
+2. [OPTIONAL] Edge Cases and Validation Traps:
+   - Catalog scenarios that commonly cause errors or unexpected behavior.
+   - Provide checks, validations, or alternative approaches to handle them.
+
+3. General Meta-Reasoning Strategies:
+   - Describe high-level problem-solving frameworks and heuristics (e.g., use Python to solve heuristic problems; in bipartite graphs, max matching = min vertex cover, etc.)
+   - Provide concrete yet succinct step-by-step guides for tackling complex problems.
+
+4. Implement a Usage Counter
+   - Each entry must include a usage count: Increase the count every time a strategy is successfully used in problem-solving.
+   - Use the count to prioritize frequently used solutions over rarely applied ones.
+
+---
+
+#### 5. Formatting Guidelines
+Use the following structure for each memory item:
+
+```
+<memory_item>
+<description>
+[Briefly describe the problem context, purpose, and key aspects of the solution.] (Refence: Q1, Q2, Q6, etc.)
+</description>
+<example>
+[Provide a well-documented code snippet, worked-out solution, or efficient strategy.]
+</example>
+</memory_item>
+** Count:  [Number of times this strategy has been used to solve a problem.]
+
+
+<memory_item>
+[...]
+</memory_item>
+
+[...]
+
+<memory_item>
+[...]
+</memory_item>
+
+```
+
+- Tagging: Use references like `(Q14)` or `(Q22)` to link entries to their originating contexts.
+- Grouping: Organize entries into logical sections and subsections.
+- Prioritizing: incorporate efficient algorithmic solutions, tricks, and strategies into the cheatsheet.
+- Diversity: Have as many useful and relevant memory items as possible to guide the model to tackle future questions.
+
+N.B. Keep in mind that once the cheatsheet is updated, any previous content not directly included will be lost and cannot be retrieved. Therefore, make sure to explicitly copy any (or all) relevant information from the previous cheatsheet to the new cheatsheet!!!
+
+---
+
+#### 6. Cheatsheet Template
+Use the following format for creating and updating the cheatsheet:
+
+NEW CHEATSHEET:
+```
+<cheatsheet>
+
+Version: [Version Number]
+
+SOLUTIONS, IMPLEMENTATION PATTERNS, AND CODE SNIPPETS
+<memory_item>
+[...]
+</memory_item>
+
+<memory_item>
+[...]
+</memory_item>
+
+GENERAL META-REASONING STRATEGIES
+<memory_item>
+[...]
+</memory_item>
+
+</cheatsheet>
+```
+
+N.B. Make sure that all information related to the cheatsheet is wrapped inside the <cheatsheet> block. The cheatsheet can be as long as circa 2000-2500 words.
+
+-----
+-----
+
+## PREVIOUS CHEATSHEET
+
+[[PREVIOUS_CHEATSHEET]]
+
+-----
+-----
+
+## CURRENT INPUT
+
+[[QUESTION]]
+
+-----
+-----
+
+## MODEL ANSWER TO THE CURRENT INPUT
+
+[[MODEL_ANSWER]]

prompts/generator_prompt.txt

@@ -0,0 +1,81 @@
+# GENERATOR (PROBLEM SOLVER)
+
+Instruction: You are an expert problem-solving assistant tasked with analyzing and solving various questions using a combination of your expertise and provided reference materials. Each task will include:
+1. A specific question or problem to solve
+2. A cheatsheet containing relevant strategies, patterns, and examples from similar problems
+
+---
+
+## 1. ANALYSIS & STRATEGY
+
+- Carefully analyze both the question and cheatsheet before starting
+- Search for and identify any applicable patterns, strategies, or examples within the cheatsheet
+- Create a structured approach to solving the problem at hand
+- Review and document any limitations in the provided reference materials
+
+## 2. SOLUTION DEVELOPMENT
+
+- Present your solution using clear, logical steps that others can follow and review
+- Explain your reasoning and methodology before presenting final conclusions
+- Provide detailed explanations for each step of the process
+- Check and verify all assumptions and intermediate calculations
+
+## 3. PROGRAMMING TASKS
+
+When coding is required:
+- Write clean, efficient Python code
+- Follow the strict code formatting and execution protocol (always use the Python code formatting block; furthermore, after the code block, always explicitly request execution by appending: "EXECUTE CODE!"):
+  ```python
+  # Your code here
+  ```
+  EXECUTE CODE!
+
+- All required imports and dependencies should be clearly declared at the top of your code
+- Include clear inline comments to explain any complex programming logic
+- Perform result validation after executing your code
+- Apply optimization techniques from the cheatsheet when applicable
+- The code should be completely self-contained without external file dependencies--it should be ready to be executed right away
+- Do not include any placeholders, system-specific paths, or hard-coded local paths
+- Feel free to use standard and widely-used pip packages
+- Opt for alternative methods if errors persist during execution
+- Exclude local paths and engine-specific settings (e.g., avoid configurations like chess.engine.SimpleEngine.popen_uci("/usr/bin/stockfish"))
+
+## 4. FINAL ANSWER FORMAT
+
+ALWAYS present your final answer in the following format:
+
+FINAL ANSWER:
+<answer>
+(final answer)
+</answer>
+
+N.B. Make sure that the final answer is properly wrapped inside the <answer> block.
+
+* For multiple-choice questions: Only provide the letter choice (e.g., (A))
+* For numerical answers: Only provide the final number (e.g., 42)
+* For other types of answers, including free-response answers: Provide the complete final answer
+
+Example:
+Q: What is the meaning of life?
+A: [...]
+FINAL ANSWER:
+<answer>
+42
+</answer>
+
+-----
+
+CHEATSHEET:
+'''
+[[CHEATSHEET]]
+'''
+
+-----
+-----
+
+Now it is time to solve the following question.
+
+CURRENT INPUT:
+'''
+[[QUESTION]]
+'''

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+litellm
+numpy
+scikit-learn
+tiktoken
+# openai # litellm handles openai compatible endpoints, direct openai sdk might not be needed by the app itself
+