x

src/lib/benchmarks/google.ts

@@ -1,208 +1,285 @@
 import { Benchmark } from "./types";
 
 export const googleBenchmarks: Benchmark[] = [
-  {
-    model: "Gemini 2.5 Pro (Thinking-enabled, default)",
-    provider: "Google",
-    inputPrice: 2.5,
-    outputPrice: 15.0,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      livecodebench_v6: 69.0,
-      aider_polyglot: 82.2,
-      swe_bench_verified: 67.2,
-      gpqa_diamond: 86.4,
-      aime_2025: 88.0,
-      humanitys_last_exam: 21.6,
-      simpleqa: 54.0,
-      facts_grounding: 87.8,
-      global_mmlu_lite: 89.2,
-      mrcr_v2_avg_128k: 58.0,
-      mrcr_v2_pointwise_1m: 16.4,
-      mmmu: 82.0,
-      // loft_128k: 87.0,
-      // loft_1m: 69.8,
+    {
+        model: "Gemini 2.5 Pro (Thinking-enabled, <=200k context)",
+        provider: "Google",
+        inputPrice: 1.25,
+        outputPrice: 10.0,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 69.0,
+            aider_polyglot: 82.2,
+            swe_bench_verified: 67.2,
+            gpqa_diamond: 86.4,
+            aime_2025: 88.0,
+            humanitys_last_exam: 21.6,
+            simpleqa: 54.0,
+            facts_grounding: 87.8,
+            global_mmlu_lite: 89.2,
+            mrcr_v2_avg_128k: 58.0,
+            mrcr_v2_pointwise_1m: 16.4,
+            mmmu: 82.0,
+        },
     },
-  },
-  {
-    model: "Gemini 2.5 Flash (Thinking-enabled, default)",
-    provider: "Google",
-    inputPrice: 0.15,
-    outputPrice: 3.5,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      livecodebench_v6: 55.4,
-      aider_polyglot: 56.7,
-      swe_bench_verified: 60.3,
-      gpqa_diamond: 82.8,
-      aime_2025: 72.0,
-      humanitys_last_exam: 11.0,
-      simpleqa: 26.9,
-      facts_grounding: 85.3,
-      global_mmlu_lite: 88.4,
-      mrcr_v2_avg_128k: 54.3,
-      mrcr_v2_pointwise_1m: 21.0,
-      mmmu: 79.7,
-      // loft_128k: 82.1,
-      // loft_1m: 58.9,
+    {
+        model: "Gemini 2.5 Pro (Thinking-enabled, >200k context)",
+        provider: "Google",
+        inputPrice: 2.5,
+        outputPrice: 15.0,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 69.0,
+            aider_polyglot: 82.2,
+            swe_bench_verified: 67.2,
+            gpqa_diamond: 86.4,
+            aime_2025: 88.0,
+            humanitys_last_exam: 21.6,
+            simpleqa: 54.0,
+            facts_grounding: 87.8,
+            global_mmlu_lite: 89.2,
+            mrcr_v2_avg_128k: 58.0,
+            mrcr_v2_pointwise_1m: 16.4,
+            mmmu: 82.0,
+        },
     },
-  },
-  {
-    model: "Gemini 2.5 Flash (Non-Thinking)",
-    provider: "Google",
-    inputPrice: 0.30,
-    outputPrice: 2.50,
-    source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
-    benchmark: {
-      humanitys_last_exam: 8.4,
-      gpqa_diamond: 78.3,
-      aime_2025: 61.6,
-      livecodebench_v6: 41.1,
-      aider_polyglot: 44.0,
-      swe_bench_verified: 50.0,
-      simpleqa: 25.8,
-      facts_grounding: 83.4,
-      mmmu: 76.9,
-      // vibe_eval: 66.2,
-      mrcr_v2_avg_128k: 34.1,
-      mrcr_v2_pointwise_1m: 16.8,
-      global_mmlu_lite: 85.8,
-      // loft_128k: 76.2,
-      // loft_1m: 49.5,
+    {
+        model: "Gemini 2.5 Flash (Thinking-enabled, default)",
+        provider: "Google",
+        inputPrice: 0.15,
+        outputPrice: 3.5,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 55.4,
+            aider_polyglot: 56.7,
+            swe_bench_verified: 60.3,
+            gpqa_diamond: 82.8,
+            aime_2025: 72.0,
+            humanitys_last_exam: 11.0,
+            simpleqa: 26.9,
+            facts_grounding: 85.3,
+            global_mmlu_lite: 88.4,
+            mrcr_v2_avg_128k: 54.3,
+            mrcr_v2_pointwise_1m: 21.0,
+            mmmu: 79.7,
+            // loft_128k: 82.1,
+            // loft_1m: 58.9,
+        },
     },
-  },
-  {
-    model: "Gemini 2.5 Flash-Lite (Non-Thinking)",
-    provider: "Google",
-    inputPrice: 0.10,
-    outputPrice: 0.40,
-    source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
-    benchmark: {
-      humanitys_last_exam: 5.1,
-      gpqa_diamond: 64.6,
-      aime_2025: 49.8,
-      livecodebench_v6: 33.7,
-      aider_polyglot: 26.7,
-      swe_bench_verified: 42.6,
-      simpleqa: 10.7,
-      facts_grounding: 84.1,
-      mmmu: 72.9,
-      // vibe_eval: 51.3,
-      mrcr_v2_avg_128k: 16.6,
-      mrcr_v2_pointwise_1m: 4.1,
-      global_mmlu_lite: 81.1,
-      // loft_128k: 65.7,
-      // loft_1m: 31.1,
+    {
+        model: "Gemini 2.5 Flash (Non-Thinking)",
+        provider: "Google",
+        inputPrice: 0.30,
+        outputPrice: 2.50,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
+        benchmark: {
+            humanitys_last_exam: 8.4,
+            gpqa_diamond: 78.3,
+            aime_2025: 61.6,
+            livecodebench_v6: 41.1,
+            aider_polyglot: 44.0,
+            swe_bench_verified: 50.0,
+            simpleqa: 25.8,
+            facts_grounding: 83.4,
+            mmmu: 76.9,
+            // vibe_eval: 66.2,
+            mrcr_v2_avg_128k: 34.1,
+            mrcr_v2_pointwise_1m: 16.8,
+            global_mmlu_lite: 85.8,
+            // loft_128k: 76.2,
+            // loft_1m: 49.5,
+        },
     },
-  },
-  {
-    model: "Gemini 2.5 Flash-Lite (Thinking)",
-    provider: "Google",
-    inputPrice: 0.10,
-    outputPrice: 0.40,
-    source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
-    benchmark: {
-      humanitys_last_exam: 6.9,
-      gpqa_diamond: 66.7,
-      aime_2025: 63.1,
-      livecodebench_v6: 34.3,
-      aider_polyglot: 27.1,
-      swe_bench_verified: 44.9,
-      simpleqa: 13.0,
-      facts_grounding: 86.8,
-      mmmu: 72.9,
-      // vibe_eval: 57.5,
-      mrcr_v2_avg_128k: 30.6,
-      mrcr_v2_pointwise_1m: 5.4,
-      global_mmlu_lite: 84.5,
-      // loft_128k: 67.3,
-      // loft_1m: 38.4,
+    {
+        model: "Gemini 2.5 Flash-Lite (Non-Thinking)",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
+        benchmark: {
+            humanitys_last_exam: 5.1,
+            gpqa_diamond: 64.6,
+            aime_2025: 49.8,
+            livecodebench_v6: 33.7,
+            aider_polyglot: 26.7,
+            swe_bench_verified: 42.6,
+            simpleqa: 10.7,
+            facts_grounding: 84.1,
+            mmmu: 72.9,
+            // vibe_eval: 51.3,
+            mrcr_v2_avg_128k: 16.6,
+            mrcr_v2_pointwise_1m: 4.1,
+            global_mmlu_lite: 81.1,
+            // loft_128k: 65.7,
+            // loft_1m: 31.1,
+        },
     },
-  },
-  {
-    model: "Gemini 2.0 Flash-Lite",
-    provider: "Google",
-    inputPrice: 0.10,
-    outputPrice: 0.40,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      livecodebench_v6: 29.1,
-      aider_polyglot: 10.5,
-      swe_bench_verified: 23.1,
-      gpqa_diamond: 50.5,
-      aime_2025: 23.8,
-      humanitys_last_exam: 4.6,
-      simpleqa: 16.5,
-      facts_grounding: 82.4,
-      global_mmlu_lite: 78.0,
-      // loft_128k: 50.7,
-      // loft_1m: 7.6,
+    {
+        model: "Gemini 2.5 Flash-Lite (Thinking)",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
+        benchmark: {
+            humanitys_last_exam: 6.9,
+            gpqa_diamond: 66.7,
+            aime_2025: 63.1,
+            livecodebench_v6: 34.3,
+            aider_polyglot: 27.1,
+            swe_bench_verified: 44.9,
+            simpleqa: 13.0,
+            facts_grounding: 86.8,
+            mmmu: 72.9,
+            // vibe_eval: 57.5,
+            mrcr_v2_avg_128k: 30.6,
+            mrcr_v2_pointwise_1m: 5.4,
+            global_mmlu_lite: 84.5,
+            // loft_128k: 67.3,
+            // loft_1m: 38.4,
+        },
     },
-  },
-  {
-    model: "Gemini 2.0 Flash",
-    provider: "Google",
-    inputPrice: 0.1,
-    outputPrice: 0.4,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      aime_2025: 29.7,
-      gpqa_diamond: 65.2,
-      simpleqa: 29.9,
-      global_mmlu_lite: 83.4,
-      livecodebench_v6: 29.1,
-      mmmu: 69.3,
-      facts_grounding: 84.6,
-      humanitys_last_exam: 5.1,
-      mrcr_v2_avg_128k: 19.0,
-      mrcr_v2_pointwise_1m: 5.3,
-      // loft_128k: 58.0,
-      // loft_1m: 7.6,
+    {
+        model: "Gemini 2.0 Flash-Lite",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 29.1,
+            aider_polyglot: 10.5,
+            swe_bench_verified: 23.1,
+            gpqa_diamond: 50.5,
+            aime_2025: 23.8,
+            humanitys_last_exam: 4.6,
+            simpleqa: 16.5,
+            facts_grounding: 82.4,
+            global_mmlu_lite: 78.0,
+            // loft_128k: 50.7,
+            // loft_1m: 7.6,
+        },
     },
-  },
-  {
-    model: "Gemini 1.5 Pro",
-    provider: "Google",
-    inputPrice: 0.015,
-    outputPrice: 0.075,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      livecodebench_v6: 29.7,
-      aider_polyglot: 16.9,
-      swe_bench_verified: 34.2,
-      gpqa_diamond: 58.1,
-      aime_2025: 17.5,
-      humanitys_last_exam: 4.6,
-      simpleqa: 24.9,
-      facts_grounding: 80.0,
-      global_mmlu_lite: 80.8,
-      mrcr_v2_avg_128k: 26.2,
-      mrcr_v2_pointwise_1m: 12.1,
-      mmmu: 67.7,
-      // loft_128k: 75.9,
-      // loft_1m: 47.1,
+    {
+        model: "Gemini Diffusion",
+        provider: "Google",
+        inputPrice: 0,
+        outputPrice: 0,
+        source: "https://deepmind.google/models/gemini-diffusion/",
+        benchmark: {
+            livecodebench_v6: 30.9,
+            bigcodebench: 45.4,
+            lbpp_v2: 56.8,
+            swe_bench_verified: 22.9,
+            humaneval: 89.6,
+            mbpp: 76.0,
+            gpqa_diamond: 40.4,
+            aime_2025: 23.3,
+            bigbench_extra_hard: 15.0,
+            global_mmlu_lite: 69.1,
+        },
     },
-  },
-  {
-    model: "Gemini 1.5 Flash",
-    provider: "Google",
-    inputPrice: 0.0025,
-    outputPrice: 0.0075,
-    source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
-    benchmark: {
-      livecodebench_v6: 30.3,
-      aider_polyglot: 2.8,
-      swe_bench_verified: 19.7,
-      gpqa_diamond: 50.0,
-      aime_2025: 14.7,
-      simpleqa: 8.6,
-      facts_grounding: 82.9,
-      global_mmlu_lite: 72.5,
-      mrcr_v2_avg_128k: 18.4,
-      mrcr_v2_pointwise_1m: 10.2,
-      mmmu: 58.3,
-      // loft_128k: 67.3,
-      // loft_1m: 36.7,
+
+    {
+        model: "Gemini 2.0 Flash",
+        provider: "Google",
+        inputPrice: 0.1,
+        outputPrice: 0.4,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            aime_2025: 29.7,
+            gpqa_diamond: 65.2,
+            simpleqa: 29.9,
+            global_mmlu_lite: 83.4,
+            livecodebench_v6: 29.1,
+            mmmu: 69.3,
+            facts_grounding: 84.6,
+            humanitys_last_exam: 5.1,
+            mrcr_v2_avg_128k: 19.0,
+            mrcr_v2_pointwise_1m: 5.3,
+            // loft_128k: 58.0,
+            // loft_1m: 7.6,
+        },
+    },
+    {
+        model: "Gemini 1.5 Pro (<=128k context)",
+        provider: "Google",
+        inputPrice: 1.25,
+        outputPrice: 5.00,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 29.7,
+            aider_polyglot: 16.9,
+            swe_bench_verified: 34.2,
+            gpqa_diamond: 58.1,
+            aime_2025: 17.5,
+            humanitys_last_exam: 4.6,
+            simpleqa: 24.9,
+            facts_grounding: 80.0,
+            global_mmlu_lite: 80.8,
+            mrcr_v2_avg_128k: 26.2,
+            mrcr_v2_pointwise_1m: 12.1,
+            mmmu: 67.7,
+        },
+    },
+    {
+        model: "Gemini 1.5 Pro (>128k context)",
+        provider: "Google",
+        inputPrice: 2.50,
+        outputPrice: 10.00,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 29.7,
+            aider_polyglot: 16.9,
+            swe_bench_verified: 34.2,
+            gpqa_diamond: 58.1,
+            aime_2025: 17.5,
+            humanitys_last_exam: 4.6,
+            simpleqa: 24.9,
+            facts_grounding: 80.0,
+            global_mmlu_lite: 80.8,
+            mrcr_v2_avg_128k: 26.2,
+            mrcr_v2_pointwise_1m: 12.1,
+            mmmu: 67.7,
+        },
+    },
+    // Gemini 1.5 Flash
+    {
+        model: "Gemini 1.5 Flash (<=128k context)",
+        provider: "Google",
+        inputPrice: 0.075,
+        outputPrice: 0.30,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 30.3,
+            aider_polyglot: 2.8,
+            swe_bench_verified: 19.7,
+            gpqa_diamond: 50.0,
+            aime_2025: 14.7,
+            simpleqa: 8.6,
+            facts_grounding: 82.9,
+            global_mmlu_lite: 72.5,
+            mrcr_v2_avg_128k: 18.4,
+            mrcr_v2_pointwise_1m: 10.2,
+            mmmu: 58.3,
+        },
+    },
+    {
+        model: "Gemini 1.5 Flash (>128k context)",
+        provider: "Google",
+        inputPrice: 0.15,
+        outputPrice: 0.60,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 30.3,
+            aider_polyglot: 2.8,
+            swe_bench_verified: 19.7,
+            gpqa_diamond: 50.0,
+            aime_2025: 14.7,
+            simpleqa: 8.6,
+            facts_grounding: 82.9,
+            global_mmlu_lite: 72.5,
+            mrcr_v2_avg_128k: 18.4,
+            mrcr_v2_pointwise_1m: 10.2,
+            mmmu: 58.3,
+        },
     },
-  },
 ];