Upload eval_results/AI-MO/deepseek-coder-33b-sft/aimo_v00.01/math_v2/results_2024-04-29T00-15-40.078181.json with huggingface_hub

eval_results/AI-MO/deepseek-coder-33b-sft/aimo_v00.01/math_v2/results_2024-04-29T00-15-40.078181.json

@@ -0,0 +1,389 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 656613.037969194,
+    "end_time": 693287.06641039,
+    "total_evaluation_time_secondes": "36674.02844119596",
+    "model_name": "AI-MO/deepseek-coder-33b-sft",
+    "model_sha": "8fa0a14e177aba2a2268e796ee660b668781190e",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "62.59 GB",
+    "config": null
+  },
+  "results": {
+    "custom|math_v2:algebra|0": {
+      "qem": 0.561078348778433,
+      "qem_stderr": 0.014409964561949595
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "qem": 0.33755274261603374,
+      "qem_stderr": 0.021742821642015735
+    },
+    "custom|math_v2:geometry|0": {
+      "qem": 0.3089770354906054,
+      "qem_stderr": 0.021134666193853444
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "qem": 0.16279069767441862,
+      "qem_stderr": 0.012292169664509116
+    },
+    "custom|math_v2:number_theory|0": {
+      "qem": 0.3,
+      "qem_stderr": 0.01973855084879312
+    },
+    "custom|math_v2:prealgebra|0": {
+      "qem": 0.5729047072330654,
+      "qem_stderr": 0.016770420856522812
+    },
+    "custom|math_v2:precalculus|0": {
+      "qem": 0.23443223443223443,
+      "qem_stderr": 0.018146891051816633
+    },
+    "custom|math_v2:_average|0": {
+      "qem": 0.35396225231782724,
+      "qem_stderr": 0.017747926402780063
+    },
+    "all": {
+      "qem": 0.35396225231782724,
+      "qem_stderr": 0.017747926402780063
+    }
+  },
+  "versions": {
+    "custom|math_v2:algebra|0": 0,
+    "custom|math_v2:counting_and_probability|0": 0,
+    "custom|math_v2:geometry|0": 0,
+    "custom|math_v2:intermediate_algebra|0": 0,
+    "custom|math_v2:number_theory|0": 0,
+    "custom|math_v2:prealgebra|0": 0,
+    "custom|math_v2:precalculus|0": 0
+  },
+  "config_tasks": {
+    "custom|math_v2:algebra": {
+      "name": "math_v2:algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:counting_and_probability": {
+      "name": "math_v2:counting_and_probability",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:geometry": {
+      "name": "math_v2:geometry",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:intermediate_algebra": {
+      "name": "math_v2:intermediate_algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:number_theory": {
+      "name": "math_v2:number_theory",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:prealgebra": {
+      "name": "math_v2:prealgebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:precalculus": {
+      "name": "math_v2:precalculus",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    }
+  },
+  "summary_tasks": {
+    "custom|math_v2:algebra|0": {
+      "hashes": {
+        "hash_examples": "6ec951c5aa417d2a",
+        "hash_full_prompts": "4e0b4b752e408da7",
+        "hash_input_tokens": "012c729bf14bf34f",
+        "hash_cont_tokens": "58352d3da7cbec27"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 176,
+      "non_padded": 1011,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "cd34cb03dc09e1ad",
+        "hash_full_prompts": "9ebbc169a1089204",
+        "hash_input_tokens": "c42a5434b2a234de",
+        "hash_cont_tokens": "165794065ea800f2"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 40,
+      "non_padded": 434,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:geometry|0": {
+      "hashes": {
+        "hash_examples": "e1011f83d0cb54d0",
+        "hash_full_prompts": "e6e4d9f63cdecf28",
+        "hash_input_tokens": "18e3fc70a490b437",
+        "hash_cont_tokens": "e94e94bbc4291776"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 133,
+      "non_padded": 346,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "aa72155be072b11c",
+        "hash_full_prompts": "38dad98a61e4f0ed",
+        "hash_input_tokens": "5970d330084eff39",
+        "hash_cont_tokens": "59489a5caaefec9b"
+      },
+      "truncated": 900,
+      "non_truncated": 3,
+      "padded": 42,
+      "non_padded": 861,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:number_theory|0": {
+      "hashes": {
+        "hash_examples": "b8565befcdbe9247",
+        "hash_full_prompts": "8ca7a4c78d6e1280",
+        "hash_input_tokens": "4ca0e34c6fbe5ff2",
+        "hash_cont_tokens": "282a36f8f1b14ec8"
+      },
+      "truncated": 540,
+      "non_truncated": 0,
+      "padded": 6,
+      "non_padded": 534,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "e04d1527fe369f16",
+        "hash_full_prompts": "51e0c4e3d62158e4",
+        "hash_input_tokens": "d9a80dc7486fcf80",
+        "hash_cont_tokens": "a5f596ebe323c507"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 54,
+      "non_padded": 817,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:precalculus|0": {
+      "hashes": {
+        "hash_examples": "97606c134f223253",
+        "hash_full_prompts": "2438e95fc348976e",
+        "hash_input_tokens": "e9c4d7582c8aa955",
+        "hash_cont_tokens": "0011317cc9b80477"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 17,
+      "non_padded": 529,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "de5f0d623d1896c2",
+      "hash_full_prompts": "3cb45e93275657cb",
+      "hash_input_tokens": "29ef0b8689a1bea3",
+      "hash_cont_tokens": "001e8eba9f86abdc"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 468,
+    "non_padded": 4532,
+    "num_truncated_few_shots": 0
+  }
+}