adding-more-google-datasets (#3)

- x (a4caafda1889a27ebe7290adee04302698a0cb28)
- x (85d193be99d010109225246baa821698cd184c54)
- x (de1d6b599d470dd072df99f667639daf296250ab)
- x (0f8dbb03a69debab4b246548c0d81c938b94b98b)
- x (9772fe4fa02854215c5867de605ebec90872d018)

src/App.tsx

@@ -104,26 +104,21 @@ const App: React.FC = () => {
 
 
 
-  const filteredBenchmarkedModels = useMemo(() => {
-    return benchmarkedModels.filter((model) => {
-      const providerMatch =
-        selectedBenchmarkProviders.length === 0 || selectedBenchmarkProviders.includes(model.provider);
-      const modelMatch =
-        selectedBenchmarkModels.length === 0 || selectedBenchmarkModels.includes(model.name);
-
-      // When not linking, allow filtering by either
-      if (selectedBenchmarkProviders.length > 0 && selectedBenchmarkModels.length > 0) {
-        return providerMatch || modelMatch;
-      }
-
-      return providerMatch && modelMatch; // this handles the case where one or both are empty
-    });
-  }, [
-    benchmarkedModels,
-    selectedBenchmarkProviders,
-    selectedBenchmarkModels,
+const filteredBenchmarkedModels = useMemo(() => {
+  return benchmarkedModels.filter((model) => {
+    const providerMatch =
+      selectedBenchmarkProviders.length === 0 || selectedBenchmarkProviders.includes(model.provider);
+    const modelMatch =
+      selectedBenchmarkModels.length === 0 || selectedBenchmarkModels.includes(model.name);
+
+    return providerMatch && modelMatch;
+  });
+}, [
+  benchmarkedModels,
+  selectedBenchmarkProviders,
+  selectedBenchmarkModels,
+]);
 
-  ]);
 
   const sortedBenchmarkedModels = useMemo(() => {
     if (!benchmarkSortConfig) return filteredBenchmarkedModels;

src/components/BenchmarkTable.tsx

@@ -64,9 +64,9 @@ export const BenchmarkTable: React.FC<BenchmarkTableProps> = ({
             }))
         );
 
-const filtered = flat.filter((m) =>
-  !selectedProviders.length || selectedProviders.includes(m.provider)
-);
+        const filtered = flat.filter((m) =>
+            !selectedProviders.length || selectedProviders.includes(m.provider)
+        );
 
 
         return Array.from(new Map(filtered.map((m) => [m.value, m])).values());

src/lib/benchmarks/google.ts

@@ -1,153 +1,285 @@
 import { Benchmark } from "./types";
 
-
 export const googleBenchmarks: Benchmark[] = [
     {
-        model: "Gemini Diffusion",
+        model: "Gemini 2.5 Pro (Thinking-enabled, <=200k context)",
         provider: "Google",
-        inputPrice: 0,
-        outputPrice: 0,
+        inputPrice: 1.25,
+        outputPrice: 10.0,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            livecodebench_v6: 30.9,
-            bigcodebench: 45.4,
-            lbpp_v2: 56.8,
-            swe_bench_verified: 22.9,
-            humaneval: 89.6,
-            mbpp: 76.0,
-            gpqa_diamond: 40.4,
-            aime_2025: 23.3,
-            bigbench_extra_hard: 15.0,
-            global_mmlu_lite: 69.1,
+            livecodebench_v6: 69.0,
+            aider_polyglot: 82.2,
+            swe_bench_verified: 67.2,
+            gpqa_diamond: 86.4,
+            aime_2025: 88.0,
+            humanitys_last_exam: 21.6,
+            simpleqa: 54.0,
+            facts_grounding: 87.8,
+            global_mmlu_lite: 89.2,
+            mrcr_v2_avg_128k: 58.0,
+            mrcr_v2_pointwise_1m: 16.4,
+            mmmu: 82.0,
         },
-        source: "https://deepmind.google/models/gemini-diffusion/",
     },
     {
-        model: "Gemini 2.0 Flash-Lite",
+        model: "Gemini 2.5 Pro (Thinking-enabled, >200k context)",
         provider: "Google",
-        inputPrice: 0.10,
-        outputPrice: 0.40,
+        inputPrice: 2.5,
+        outputPrice: 15.0,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            livecodebench_v6: 28.5,
-            bigcodebench: 45.8,
-            lbpp_v2: 56.0,
-            swe_bench_verified: 28.5,
-            humaneval: 90.2,
-            mbpp: 75.8,
-            gpqa_diamond: 56.5,
-            aime_2025: 20.0,
-            bigbench_extra_hard: 21.0,
-            global_mmlu_lite: 79.0,
+            livecodebench_v6: 69.0,
+            aider_polyglot: 82.2,
+            swe_bench_verified: 67.2,
+            gpqa_diamond: 86.4,
+            aime_2025: 88.0,
+            humanitys_last_exam: 21.6,
+            simpleqa: 54.0,
+            facts_grounding: 87.8,
+            global_mmlu_lite: 89.2,
+            mrcr_v2_avg_128k: 58.0,
+            mrcr_v2_pointwise_1m: 16.4,
+            mmmu: 82.0,
         },
-        source: "https://deepmind.google/models/gemini-diffusion/",
     },
-
     {
-        model: "Gemini 2.5 Flash Preview (05-20)",
+        model: "Gemini 2.5 Flash (Thinking-enabled, default)",
         provider: "Google",
         inputPrice: 0.15,
         outputPrice: 3.5,
-        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            aime_2025: 72.0,
+            livecodebench_v6: 55.4,
+            aider_polyglot: 56.7,
+            swe_bench_verified: 60.3,
             gpqa_diamond: 82.8,
+            aime_2025: 72.0,
+            humanitys_last_exam: 11.0,
             simpleqa: 26.9,
+            facts_grounding: 85.3,
             global_mmlu_lite: 88.4,
-            swe_bench_verified: 60.4,
-            livecodebench_v6: 63.9,
+            mrcr_v2_avg_128k: 54.3,
+            mrcr_v2_pointwise_1m: 21.0,
             mmmu: 79.7,
-            lbpp_v2: 61.9,
-            bigcodebench: 56.7,
-            facts_grounding: 85.3,
-            humanitys_last_exam: 11.0,
-            mrcr_v2_avg_128k: 74.0,
-            mrcr_v2_pointwise_1m: 32.0,
-
+            // loft_128k: 82.1,
+            // loft_1m: 58.9,
         },
     },
     {
-        model: "Gemini 2.5 Flash Preview (04-17) Thinking",
+        model: "Gemini 2.5 Flash (Non-Thinking)",
         provider: "Google",
-        inputPrice: 0.15,
-        outputPrice: 3.5,
-        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        inputPrice: 0.30,
+        outputPrice: 2.50,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
         benchmark: {
-            aime_2025: 78.0,
+            humanitys_last_exam: 8.4,
             gpqa_diamond: 78.3,
-            simpleqa: 29.7,
-            global_mmlu_lite: 88.4,
-            livecodebench_v6: 63.5,
-            lbpp_v2: 51.1,
-            bigcodebench: 44.2,
-            mmmu: 76.7,
-            humanitys_last_exam: 12.1
-
+            aime_2025: 61.6,
+            livecodebench_v6: 41.1,
+            aider_polyglot: 44.0,
+            swe_bench_verified: 50.0,
+            simpleqa: 25.8,
+            facts_grounding: 83.4,
+            mmmu: 76.9,
+            // vibe_eval: 66.2,
+            mrcr_v2_avg_128k: 34.1,
+            mrcr_v2_pointwise_1m: 16.8,
+            global_mmlu_lite: 85.8,
+            // loft_128k: 76.2,
+            // loft_1m: 49.5,
+        },
+    },
+    {
+        model: "Gemini 2.5 Flash-Lite (Non-Thinking)",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
+        benchmark: {
+            humanitys_last_exam: 5.1,
+            gpqa_diamond: 64.6,
+            aime_2025: 49.8,
+            livecodebench_v6: 33.7,
+            aider_polyglot: 26.7,
+            swe_bench_verified: 42.6,
+            simpleqa: 10.7,
+            facts_grounding: 84.1,
+            mmmu: 72.9,
+            // vibe_eval: 51.3,
+            mrcr_v2_avg_128k: 16.6,
+            mrcr_v2_pointwise_1m: 4.1,
+            global_mmlu_lite: 81.1,
+            // loft_128k: 65.7,
+            // loft_1m: 31.1,
+        },
+    },
+    {
+        model: "Gemini 2.5 Flash-Lite (Thinking)",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/",
+        benchmark: {
+            humanitys_last_exam: 6.9,
+            gpqa_diamond: 66.7,
+            aime_2025: 63.1,
+            livecodebench_v6: 34.3,
+            aider_polyglot: 27.1,
+            swe_bench_verified: 44.9,
+            simpleqa: 13.0,
+            facts_grounding: 86.8,
+            mmmu: 72.9,
+            // vibe_eval: 57.5,
+            mrcr_v2_avg_128k: 30.6,
+            mrcr_v2_pointwise_1m: 5.4,
+            global_mmlu_lite: 84.5,
+            // loft_128k: 67.3,
+            // loft_1m: 38.4,
         },
     },
+    {
+        model: "Gemini 2.0 Flash-Lite",
+        provider: "Google",
+        inputPrice: 0.10,
+        outputPrice: 0.40,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 29.1,
+            aider_polyglot: 10.5,
+            swe_bench_verified: 23.1,
+            gpqa_diamond: 50.5,
+            aime_2025: 23.8,
+            humanitys_last_exam: 4.6,
+            simpleqa: 16.5,
+            facts_grounding: 82.4,
+            global_mmlu_lite: 78.0,
+            // loft_128k: 50.7,
+            // loft_1m: 7.6,
+        },
+    },
+    {
+        model: "Gemini Diffusion",
+        provider: "Google",
+        inputPrice: 0,
+        outputPrice: 0,
+        source: "https://deepmind.google/models/gemini-diffusion/",
+        benchmark: {
+            livecodebench_v6: 30.9,
+            bigcodebench: 45.4,
+            lbpp_v2: 56.8,
+            swe_bench_verified: 22.9,
+            humaneval: 89.6,
+            mbpp: 76.0,
+            gpqa_diamond: 40.4,
+            aime_2025: 23.3,
+            bigbench_extra_hard: 15.0,
+            global_mmlu_lite: 69.1,
+        },
+    },
+
     {
         model: "Gemini 2.0 Flash",
         provider: "Google",
         inputPrice: 0.1,
         outputPrice: 0.4,
-        source: "https://ai.google.dev/gemini-api/docs/thinking",
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            aime_2025: 27.5,
-            gpqa_diamond: 60.1,
+            aime_2025: 29.7,
+            gpqa_diamond: 65.2,
             simpleqa: 29.9,
             global_mmlu_lite: 83.4,
-            livecodebench_v6: 34.5,
-            lbpp_v2: 22.2,
-            mmmu: 71.7,
+            livecodebench_v6: 29.1,
+            mmmu: 69.3,
             facts_grounding: 84.6,
             humanitys_last_exam: 5.1,
-            mrcr_v2_avg_128k: 36.0,
-            mrcr_v2_pointwise_1m: 6.0,
-
+            mrcr_v2_avg_128k: 19.0,
+            mrcr_v2_pointwise_1m: 5.3,
+            // loft_128k: 58.0,
+            // loft_1m: 7.6,
         },
     },
     {
-        model: "Gemini 2.5 Pro Preview (05-06)",
+        model: "Gemini 1.5 Pro (<=128k context)",
         provider: "Google",
-        inputPrice: 2.5,
-        outputPrice: 15.0,
-        source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/",
+        inputPrice: 1.25,
+        outputPrice: 5.00,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            humanitys_last_exam: 17.8,
-            gpqa_diamond: 83.0,
-            aime_2025: 83.0,
-            livecodebench_v6: 75.6,
-            lbpp_v2: 76.5,
-            bigcodebench: 72.7,
-            swe_bench_verified: 63.2,
-            simpleqa: 50.8,
-            mmmu: 79.6,
-
-            video_mme: 84.8,
-            mrcr_v2_avg_128k: 93.0,
-            mrcr_v2_pointwise_1m: 82.9,
-            global_mmlu_lite: 88.6,
+            livecodebench_v6: 29.7,
+            aider_polyglot: 16.9,
+            swe_bench_verified: 34.2,
+            gpqa_diamond: 58.1,
+            aime_2025: 17.5,
+            humanitys_last_exam: 4.6,
+            simpleqa: 24.9,
+            facts_grounding: 80.0,
+            global_mmlu_lite: 80.8,
+            mrcr_v2_avg_128k: 26.2,
+            mrcr_v2_pointwise_1m: 12.1,
+            mmmu: 67.7,
         },
     },
     {
-        model: "Gemini 2.5 Pro Experimental (03-25)",
+        model: "Gemini 1.5 Pro (>128k context)",
         provider: "Google",
-        inputPrice: 2.5,
-        outputPrice: 15.0,
-        source: "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/",
+        inputPrice: 2.50,
+        outputPrice: 10.00,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
         benchmark: {
-            humanitys_last_exam: 18.8,
-            gpqa_diamond: 84.0,
-            aime_2025: 86.7,
-            livecodebench_v6: 70.4,
-            lbpp_v2: 74.0,
-            bigcodebench: 68.6,
-            swe_bench_verified: 63.8,
-            simpleqa: 52.9,
-            mmmu: 81.7,
-            mrcr_v2_avg_128k: 94.5,
-            mrcr_v2_pointwise_1m: 83.1,
-            global_mmlu_lite: 89.8,
+            livecodebench_v6: 29.7,
+            aider_polyglot: 16.9,
+            swe_bench_verified: 34.2,
+            gpqa_diamond: 58.1,
+            aime_2025: 17.5,
+            humanitys_last_exam: 4.6,
+            simpleqa: 24.9,
+            facts_grounding: 80.0,
+            global_mmlu_lite: 80.8,
+            mrcr_v2_avg_128k: 26.2,
+            mrcr_v2_pointwise_1m: 12.1,
+            mmmu: 67.7,
+        },
+    },
+    // Gemini 1.5 Flash
+    {
+        model: "Gemini 1.5 Flash (<=128k context)",
+        provider: "Google",
+        inputPrice: 0.075,
+        outputPrice: 0.30,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 30.3,
+            aider_polyglot: 2.8,
+            swe_bench_verified: 19.7,
+            gpqa_diamond: 50.0,
+            aime_2025: 14.7,
+            simpleqa: 8.6,
+            facts_grounding: 82.9,
+            global_mmlu_lite: 72.5,
+            mrcr_v2_avg_128k: 18.4,
+            mrcr_v2_pointwise_1m: 10.2,
+            mmmu: 58.3,
+        },
+    },
+    {
+        model: "Gemini 1.5 Flash (>128k context)",
+        provider: "Google",
+        inputPrice: 0.15,
+        outputPrice: 0.60,
+        source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
+        benchmark: {
+            livecodebench_v6: 30.3,
+            aider_polyglot: 2.8,
+            swe_bench_verified: 19.7,
+            gpqa_diamond: 50.0,
+            aime_2025: 14.7,
+            simpleqa: 8.6,
+            facts_grounding: 82.9,
+            global_mmlu_lite: 72.5,
+            mrcr_v2_avg_128k: 18.4,
+            mrcr_v2_pointwise_1m: 10.2,
+            mmmu: 58.3,
         },
     },
-
-
 ];

src/lib/benchmarks/types.ts

@@ -13,6 +13,7 @@ export type BenchmarkMetric =
 
   // Code benchmarks (frequent)
   | "humaneval"
+  | "aider_polyglot"
   | "mbpp"
   | "bigcodebench"
   | "livecodebench_v6"
@@ -54,6 +55,7 @@ export const benchmarkMetricOrder: BenchmarkMetric[] = [
   "aime_24",
   "aime_2025",
   "gpqa_diamond",
+  "aider_polyglot"
 
   // // Code benchmarks (frequent)
   // "humaneval",