import { Benchmark } from "./types"; export const googleBenchmarks: Benchmark[] = [ { model: "Gemini 2.5 Pro (Thinking-enabled, <=200k context)", provider: "Google", inputPrice: 1.25, outputPrice: 10.0, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 69.0, aider_polyglot: 82.2, swe_bench_verified: 67.2, gpqa_diamond: 86.4, aime_2025: 88.0, humanitys_last_exam: 21.6, simpleqa: 54.0, facts_grounding: 87.8, global_mmlu_lite: 89.2, mrcr_v2_avg_128k: 58.0, mrcr_v2_pointwise_1m: 16.4, mmmu: 82.0, }, }, { model: "Gemini 2.5 Pro (Thinking-enabled, >200k context)", provider: "Google", inputPrice: 2.5, outputPrice: 15.0, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 69.0, aider_polyglot: 82.2, swe_bench_verified: 67.2, gpqa_diamond: 86.4, aime_2025: 88.0, humanitys_last_exam: 21.6, simpleqa: 54.0, facts_grounding: 87.8, global_mmlu_lite: 89.2, mrcr_v2_avg_128k: 58.0, mrcr_v2_pointwise_1m: 16.4, mmmu: 82.0, }, }, { model: "Gemini 2.5 Pro Experimental (03-25)", provider: "Google", inputPrice: 1.25, outputPrice: 10.0, source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/", benchmark: { livecodebench_v6: 70.4, aider_polyglot: 74.0, swe_bench_verified: 63.8, gpqa_diamond: 84.0, aime_2025: 86.7, humanitys_last_exam: 18.8, simpleqa: 52.9, global_mmlu_lite: 89.8, mrcr_v2_avg_128k: 94.5, mrcr_v2_pointwise_1m: 83.1, mmmu: 81.7, // vibe_eval: 69.4, // video_mme: not reported }, }, { model: "Gemini 2.5 Pro Preview (05-06)", provider: "Google", inputPrice: 1.25, outputPrice: 10.0, source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/", benchmark: { livecodebench_v6: 75.6, aider_polyglot: 76.5, swe_bench_verified: 63.2, gpqa_diamond: 83.0, aime_2025: 83.0, humanitys_last_exam: 17.8, simpleqa: 50.8, global_mmlu_lite: 88.6, mrcr_v2_avg_128k: 93.0, mrcr_v2_pointwise_1m: 82.9, mmmu: 79.6, // vibe_eval: 65.6, // video_mme: 84.8, }, }, { model: "Gemini 2.5 Flash (Thinking-enabled, default)", provider: "Google", inputPrice: 0.30, outputPrice: 3.5, source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/", benchmark: { livecodebench_v6: 55.4, aider_polyglot: 56.7, swe_bench_verified: 60.3, gpqa_diamond: 82.8, aime_2025: 72.0, humanitys_last_exam: 11.0, simpleqa: 26.9, facts_grounding: 85.3, global_mmlu_lite: 88.4, mrcr_v2_avg_128k: 54.3, mrcr_v2_pointwise_1m: 21.0, mmmu: 79.7, // loft_128k: 82.1, // loft_1m: 58.9, }, }, { model: "Gemini 2.5 Flash (Non-Thinking)", provider: "Google", inputPrice: 0.30, outputPrice: 2.50, source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/", benchmark: { humanitys_last_exam: 8.4, gpqa_diamond: 78.3, aime_2025: 61.6, livecodebench_v6: 41.1, aider_polyglot: 44.0, swe_bench_verified: 50.0, simpleqa: 25.8, facts_grounding: 83.4, mmmu: 76.9, // vibe_eval: 66.2, mrcr_v2_avg_128k: 34.1, mrcr_v2_pointwise_1m: 16.8, global_mmlu_lite: 85.8, // loft_128k: 76.2, // loft_1m: 49.5, }, }, { model: "Gemini 2.5 Flash-Lite (Non-Thinking)", provider: "Google", inputPrice: 0.10, outputPrice: 0.40, source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/", benchmark: { humanitys_last_exam: 5.1, gpqa_diamond: 64.6, aime_2025: 49.8, livecodebench_v6: 33.7, aider_polyglot: 26.7, swe_bench_verified: 42.6, simpleqa: 10.7, facts_grounding: 84.1, mmmu: 72.9, // vibe_eval: 51.3, mrcr_v2_avg_128k: 16.6, mrcr_v2_pointwise_1m: 4.1, global_mmlu_lite: 81.1, // loft_128k: 65.7, // loft_1m: 31.1, }, }, { model: "Gemini 2.5 Flash-Lite (Thinking)", provider: "Google", inputPrice: 0.10, outputPrice: 0.40, source: "https://blog.google/products/gemini/gemini-2-5-model-family-expands/", benchmark: { humanitys_last_exam: 6.9, gpqa_diamond: 66.7, aime_2025: 63.1, livecodebench_v6: 34.3, aider_polyglot: 27.1, swe_bench_verified: 44.9, simpleqa: 13.0, facts_grounding: 86.8, mmmu: 72.9, // vibe_eval: 57.5, mrcr_v2_avg_128k: 30.6, mrcr_v2_pointwise_1m: 5.4, global_mmlu_lite: 84.5, // loft_128k: 67.3, // loft_1m: 38.4, }, }, { model: "Gemini 2.0 Flash-Lite", provider: "Google", inputPrice: 0.10, outputPrice: 0.40, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 29.1, aider_polyglot: 10.5, swe_bench_verified: 23.1, gpqa_diamond: 50.5, aime_2025: 23.8, humanitys_last_exam: 4.6, simpleqa: 16.5, facts_grounding: 82.4, global_mmlu_lite: 78.0, // loft_128k: 50.7, // loft_1m: 7.6, }, }, { model: "Gemini Diffusion", provider: "Google", inputPrice: 0, outputPrice: 0, source: "https://deepmind.google/models/gemini-diffusion/", benchmark: { livecodebench_v6: 30.9, bigcodebench: 45.4, lbpp_v2: 56.8, swe_bench_verified: 22.9, humaneval: 89.6, mbpp: 76.0, gpqa_diamond: 40.4, aime_2025: 23.3, bigbench_extra_hard: 15.0, global_mmlu_lite: 69.1, }, }, { model: "Gemini 2.0 Flash", provider: "Google", inputPrice: 0.1, outputPrice: 0.4, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { aime_2025: 29.7, gpqa_diamond: 65.2, simpleqa: 29.9, global_mmlu_lite: 83.4, livecodebench_v6: 29.1, mmmu: 69.3, facts_grounding: 84.6, humanitys_last_exam: 5.1, mrcr_v2_avg_128k: 19.0, mrcr_v2_pointwise_1m: 5.3, // loft_128k: 58.0, // loft_1m: 7.6, }, }, { model: "Gemini 1.5 Pro (<=128k context)", provider: "Google", inputPrice: 1.25, outputPrice: 5.00, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 29.7, aider_polyglot: 16.9, swe_bench_verified: 34.2, gpqa_diamond: 58.1, aime_2025: 17.5, humanitys_last_exam: 4.6, simpleqa: 24.9, facts_grounding: 80.0, global_mmlu_lite: 80.8, mrcr_v2_avg_128k: 26.2, mrcr_v2_pointwise_1m: 12.1, mmmu: 67.7, }, }, { model: "Gemini 1.5 Pro (>128k context)", provider: "Google", inputPrice: 2.50, outputPrice: 10.00, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 29.7, aider_polyglot: 16.9, swe_bench_verified: 34.2, gpqa_diamond: 58.1, aime_2025: 17.5, humanitys_last_exam: 4.6, simpleqa: 24.9, facts_grounding: 80.0, global_mmlu_lite: 80.8, mrcr_v2_avg_128k: 26.2, mrcr_v2_pointwise_1m: 12.1, mmmu: 67.7, }, }, // Gemini 1.5 Flash { model: "Gemini 1.5 Flash (<=128k context)", provider: "Google", inputPrice: 0.075, outputPrice: 0.30, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 30.3, aider_polyglot: 2.8, swe_bench_verified: 19.7, gpqa_diamond: 50.0, aime_2025: 14.7, simpleqa: 8.6, facts_grounding: 82.9, global_mmlu_lite: 72.5, mrcr_v2_avg_128k: 18.4, mrcr_v2_pointwise_1m: 10.2, mmmu: 58.3, }, }, { model: "Gemini 1.5 Flash (>128k context)", provider: "Google", inputPrice: 0.15, outputPrice: 0.60, source: "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", benchmark: { livecodebench_v6: 30.3, aider_polyglot: 2.8, swe_bench_verified: 19.7, gpqa_diamond: 50.0, aime_2025: 14.7, simpleqa: 8.6, facts_grounding: 82.9, global_mmlu_lite: 72.5, mrcr_v2_avg_128k: 18.4, mrcr_v2_pointwise_1m: 10.2, mmmu: 58.3, }, }, ];