Delete eval_results/AI-MO/deepseek-math-7b-sft/aimo_v00.00/math/results_2024-04-24T13-57-53.176523.json

eval_results/AI-MO/deepseek-math-7b-sft/aimo_v00.00/math/results_2024-04-24T13-57-53.176523.json

@@ -1,410 +0,0 @@
-{
-  "config_general": {
-    "lighteval_sha": "?",
-    "num_fewshot_seeds": 1,
-    "override_batch_size": 2,
-    "max_samples": 10,
-    "job_id": "",
-    "start_time": 3652558.79848515,
-    "end_time": 3652962.355642761,
-    "total_evaluation_time_secondes": "403.55715761100873",
-    "model_name": "AI-MO/deepseek-math-7b-sft",
-    "model_sha": "748a5f5e457052be6c9476d87222e596fedefdb7",
-    "model_dtype": "torch.bfloat16",
-    "model_size": "12.93 GB",
-    "config": null
-  },
-  "results": {
-    "lighteval|math:algebra|0": {
-      "qem": 0.7,
-      "qem_stderr": 0.15275252316519466
-    },
-    "lighteval|math:counting_and_probability|0": {
-      "qem": 0.5,
-      "qem_stderr": 0.16666666666666666
-    },
-    "lighteval|math:geometry|0": {
-      "qem": 0.3,
-      "qem_stderr": 0.15275252316519466
-    },
-    "lighteval|math:intermediate_algebra|0": {
-      "qem": 0.1,
-      "qem_stderr": 0.09999999999999999
-    },
-    "lighteval|math:number_theory|0": {
-      "qem": 0.2,
-      "qem_stderr": 0.13333333333333333
-    },
-    "lighteval|math:prealgebra|0": {
-      "qem": 0.6,
-      "qem_stderr": 0.16329931618554522
-    },
-    "lighteval|math:precalculus|0": {
-      "qem": 0.4,
-      "qem_stderr": 0.1632993161855452
-    },
-    "lighteval|math:_average|0": {
-      "qem": 0.39999999999999997,
-      "qem_stderr": 0.14744338267163995
-    },
-    "all": {
-      "qem": 0.39999999999999997,
-      "qem_stderr": 0.14744338267163995
-    }
-  },
-  "versions": {
-    "lighteval|math:algebra|0": 0,
-    "lighteval|math:counting_and_probability|0": 0,
-    "lighteval|math:geometry|0": 0,
-    "lighteval|math:intermediate_algebra|0": 0,
-    "lighteval|math:number_theory|0": 0,
-    "lighteval|math:prealgebra|0": 0,
-    "lighteval|math:precalculus|0": 0
-  },
-  "config_tasks": {
-    "lighteval|math:algebra": {
-      "name": "math:algebra",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "algebra",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 1187,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:counting_and_probability": {
-      "name": "math:counting_and_probability",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "counting_and_probability",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 474,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:geometry": {
-      "name": "math:geometry",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "geometry",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 479,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:intermediate_algebra": {
-      "name": "math:intermediate_algebra",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "intermediate_algebra",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 903,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:number_theory": {
-      "name": "math:number_theory",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "number_theory",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 540,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:prealgebra": {
-      "name": "math:prealgebra",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "prealgebra",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 871,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    },
-    "lighteval|math:precalculus": {
-      "name": "math:precalculus",
-      "prompt_function": "math",
-      "hf_repo": "lighteval/MATH",
-      "hf_subset": "precalculus",
-      "metric": [
-        "quasi_exact_match_math"
-      ],
-      "hf_avail_splits": [
-        "train",
-        "test",
-        "validation"
-      ],
-      "evaluation_splits": [
-        "test"
-      ],
-      "few_shots_split": null,
-      "few_shots_select": null,
-      "generation_size": 2048,
-      "stop_sequence": [
-        "\n"
-      ],
-      "output_regex": null,
-      "frozen": false,
-      "suite": [
-        "lighteval",
-        "math"
-      ],
-      "original_num_docs": 546,
-      "effective_num_docs": 10,
-      "trust_dataset": true,
-      "must_remove_duplicate_docs": null
-    }
-  },
-  "summary_tasks": {
-    "lighteval|math:algebra|0": {
-      "hashes": {
-        "hash_examples": "a13d68854ca927ce",
-        "hash_full_prompts": "7e0d2b25e14caad6",
-        "hash_input_tokens": "58c8826560827dfc",
-        "hash_cont_tokens": "310523472b3267fc"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 3,
-      "non_padded": 7,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:counting_and_probability|0": {
-      "hashes": {
-        "hash_examples": "a8004c36a2d9cb68",
-        "hash_full_prompts": "2acaf205499ed79c",
-        "hash_input_tokens": "d76018dbed1fcc48",
-        "hash_cont_tokens": "733b9c4cba844ec2"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 1,
-      "non_padded": 9,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:geometry|0": {
-      "hashes": {
-        "hash_examples": "5e12e37f7378cc4c",
-        "hash_full_prompts": "32e7c26bfc66828d",
-        "hash_input_tokens": "288b95f1a6a9a3d5",
-        "hash_cont_tokens": "8d9422af27507fe7"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 5,
-      "non_padded": 5,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:intermediate_algebra|0": {
-      "hashes": {
-        "hash_examples": "71738fc49d471d6d",
-        "hash_full_prompts": "6779f5c079af81a6",
-        "hash_input_tokens": "53a5702086e49106",
-        "hash_cont_tokens": "6f8722f0a58ef37a"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 3,
-      "non_padded": 7,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:number_theory|0": {
-      "hashes": {
-        "hash_examples": "bdb66471a0eed93a",
-        "hash_full_prompts": "9985b650f03b8f91",
-        "hash_input_tokens": "8b240f273f300e85",
-        "hash_cont_tokens": "ae8bd3b0e9f74ac5"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 5,
-      "non_padded": 5,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:prealgebra|0": {
-      "hashes": {
-        "hash_examples": "3c59373ec7e3a94a",
-        "hash_full_prompts": "722cea4098cecd00",
-        "hash_input_tokens": "6788548239da9c91",
-        "hash_cont_tokens": "c2f2eaca08a1e171"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 1,
-      "non_padded": 9,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    },
-    "lighteval|math:precalculus|0": {
-      "hashes": {
-        "hash_examples": "8a97d7d7bd780ca3",
-        "hash_full_prompts": "df2793e826f0dfcc",
-        "hash_input_tokens": "90b2332eeca3284b",
-        "hash_cont_tokens": "56aff6344490817d"
-      },
-      "truncated": 10,
-      "non_truncated": 0,
-      "padded": 4,
-      "non_padded": 6,
-      "effective_few_shots": 0.0,
-      "num_truncated_few_shots": 0
-    }
-  },
-  "summary_general": {
-    "hashes": {
-      "hash_examples": "e05a305d27ed0540",
-      "hash_full_prompts": "7847980860d96b71",
-      "hash_input_tokens": "a2515743ef01f5cf",
-      "hash_cont_tokens": "734c8abceb12e43b"
-    },
-    "truncated": 70,
-    "non_truncated": 0,
-    "padded": 22,
-    "non_padded": 48,
-    "num_truncated_few_shots": 0
-  }
-}