leaderboard/boards_data/summarization_PnSummary.jsonl

{"Model Name":"claude-3-7-sonnet-20250219","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1188323392,"summarization_PnSummary_rougeL_recall":0.3948447809,"summarization_PnSummary_rougeL_f1_score":0.1786530476,"nlg_score":0.1779340777}
{"Model Name":"gpt-oss:20b","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"20000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0796516779,"summarization_PnSummary_rougeL_recall":0.3573917363,"summarization_PnSummary_rougeL_f1_score":0.1263677591,"nlg_score":0.1334687319}
{"Model Name":"gemma-3-4b-it","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"4300000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1095844839,"summarization_PnSummary_rougeL_recall":0.3735331299,"summarization_PnSummary_rougeL_f1_score":0.1645385252,"nlg_score":0.0949943578}
{"Model Name":"c4ai-command-r-plus","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"104000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1227039295,"summarization_PnSummary_rougeL_recall":0.4315497639,"summarization_PnSummary_rougeL_f1_score":0.1856517383,"nlg_score":0.1880477876}
{"Model Name":"gpt-5-mini","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.0917305447,"summarization_PnSummary_rougeL_recall":0.3893845098,"summarization_PnSummary_rougeL_f1_score":0.1447284086,"nlg_score":0.181552926}
{"Model Name":"gpt-5-nano","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.0830986853,"summarization_PnSummary_rougeL_recall":0.3565850313,"summarization_PnSummary_rougeL_f1_score":0.1308633101,"nlg_score":0.1643361642}
{"Model Name":"gpt-oss:120b","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"20000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0783955467,"summarization_PnSummary_rougeL_recall":0.3671775126,"summarization_PnSummary_rougeL_f1_score":0.1256898612,"nlg_score":0.1454089411}
{"Model Name":"gemma-3n-E4B-it","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"7850000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1072486168,"summarization_PnSummary_rougeL_recall":0.3966587345,"summarization_PnSummary_rougeL_f1_score":0.1627029568,"nlg_score":0.0940241349}
{"Model Name":"gpt-4.1","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1234743619,"summarization_PnSummary_rougeL_recall":0.376111826,"summarization_PnSummary_rougeL_f1_score":0.1808600563,"nlg_score":0.194675133}
{"Model Name":"o4-mini","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":null,"summarization_PnSummary_rougeL_recall":null,"summarization_PnSummary_rougeL_f1_score":null,"nlg_score":null}
{"Model Name":"gemma-3-12b-it","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"12200000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1156871575,"summarization_PnSummary_rougeL_recall":0.3630716995,"summarization_PnSummary_rougeL_f1_score":0.1697348346,"nlg_score":0.1196804312}
{"Model Name":"gemma-3-27b-it","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"27400000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1124574222,"summarization_PnSummary_rougeL_recall":0.3717393409,"summarization_PnSummary_rougeL_f1_score":0.1673025553,"nlg_score":0.1067134448}
{"Model Name":"Qwen3-14B","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"14800000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1123870374,"summarization_PnSummary_rougeL_recall":0.4032007327,"summarization_PnSummary_rougeL_f1_score":0.17115848,"nlg_score":0.16056333}
{"Model Name":"Qwen3-32B","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"32800000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1089978489,"summarization_PnSummary_rougeL_recall":0.3936021933,"summarization_PnSummary_rougeL_f1_score":0.1662525669,"nlg_score":0.1679338638}
{"Model Name":"llama4:scout","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"109000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1377053282,"summarization_PnSummary_rougeL_recall":0.380073051,"summarization_PnSummary_rougeL_f1_score":0.1928750247,"nlg_score":0.1567965528}
{"Model Name":"claude-3-5-haiku-20241022","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.132916538,"summarization_PnSummary_rougeL_recall":0.3579358655,"summarization_PnSummary_rougeL_f1_score":0.1887379797,"nlg_score":0.1089333827}
{"Model Name":"Mistral-Small-3.1-24B-Instruct-2503","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"24000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1029257145,"summarization_PnSummary_rougeL_recall":0.4347811424,"summarization_PnSummary_rougeL_f1_score":0.1621438757,"nlg_score":0.1319091735}
{"Model Name":"DeepSeek-R1-0528-Qwen3-8B","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"8190000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0833149314,"summarization_PnSummary_rougeL_recall":0.4027758903,"summarization_PnSummary_rougeL_f1_score":0.1338404051,"nlg_score":0.112015688}
{"Model Name":"deepseek-chat","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"671000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0811186656,"summarization_PnSummary_rougeL_recall":0.3940089293,"summarization_PnSummary_rougeL_f1_score":0.1316106196,"nlg_score":0.0934094344}
{"Model Name":"Qwen3-4B","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"4020000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1067208324,"summarization_PnSummary_rougeL_recall":0.4109136551,"summarization_PnSummary_rougeL_f1_score":0.1648475797,"nlg_score":0.1389297212}
{"Model Name":"gemma-3-1b-it","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"1000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.111370713,"summarization_PnSummary_rougeL_recall":0.3732014316,"summarization_PnSummary_rougeL_f1_score":0.1661125342,"nlg_score":0.0682994522}
{"Model Name":"aya-expanse-32b","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"32300000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1094933227,"summarization_PnSummary_rougeL_recall":0.3979476995,"summarization_PnSummary_rougeL_f1_score":0.1674664883,"nlg_score":0.1196400535}
{"Model Name":"Llama-3.3-70B-Instruct","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"70600000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.2237919051,"summarization_PnSummary_rougeL_recall":0.3532978852,"summarization_PnSummary_rougeL_f1_score":0.2484855426,"nlg_score":0.2010896964}
{"Model Name":"gpt-4.1-mini","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1156493376,"summarization_PnSummary_rougeL_recall":0.403347998,"summarization_PnSummary_rougeL_f1_score":0.1750055649,"nlg_score":0.1901206806}
{"Model Name":"o3","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.0950296419,"summarization_PnSummary_rougeL_recall":0.3412128078,"summarization_PnSummary_rougeL_f1_score":0.1438085772,"nlg_score":0.1764906292}
{"Model Name":"gpt-4o-mini","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1160048958,"summarization_PnSummary_rougeL_recall":0.3980422927,"summarization_PnSummary_rougeL_f1_score":0.1751797476,"nlg_score":0.1810678527}
{"Model Name":"c4ai-command-a-03-2025","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"111000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1116612356,"summarization_PnSummary_rougeL_recall":0.420922163,"summarization_PnSummary_rougeL_f1_score":0.1723099731,"nlg_score":0.1137933652}
{"Model Name":"gemini-2.0-flash","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1228424623,"summarization_PnSummary_rougeL_recall":0.3750771332,"summarization_PnSummary_rougeL_f1_score":0.1793201723,"nlg_score":0.178231145}
{"Model Name":"gemini-2.5-flash","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1070140366,"summarization_PnSummary_rougeL_recall":0.4357356292,"summarization_PnSummary_rougeL_f1_score":0.1672508999,"nlg_score":0.1368740087}
{"Model Name":"gemini-2.0-flash-lite","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1191404443,"summarization_PnSummary_rougeL_recall":0.365434541,"summarization_PnSummary_rougeL_f1_score":0.1744092468,"nlg_score":0.1659339021}
{"Model Name":"c4ai-command-r-v01","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"35000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1298447221,"summarization_PnSummary_rougeL_recall":0.3548911672,"summarization_PnSummary_rougeL_f1_score":0.1841564462,"nlg_score":0.1641995602}
{"Model Name":"gpt-4.1-nano","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1120916238,"summarization_PnSummary_rougeL_recall":0.3610411286,"summarization_PnSummary_rougeL_f1_score":0.1660826543,"nlg_score":0.1665903777}
{"Model Name":"Qwen3-8B","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"8190000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.109255822,"summarization_PnSummary_rougeL_recall":0.3979273385,"summarization_PnSummary_rougeL_f1_score":0.1669061111,"nlg_score":0.1557270864}
{"Model Name":"Mistral-7B-Instruct-v0.3","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"7250000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0921640152,"summarization_PnSummary_rougeL_recall":0.4401953868,"summarization_PnSummary_rougeL_f1_score":0.1480945013,"nlg_score":0.0944140383}
{"Model Name":"gpt-4o","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1134979628,"summarization_PnSummary_rougeL_recall":0.3909794734,"summarization_PnSummary_rougeL_f1_score":0.1716841943,"nlg_score":0.18964968}
{"Model Name":"deepseek-reasoner","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"671000000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.0849469928,"summarization_PnSummary_rougeL_recall":0.3820724231,"summarization_PnSummary_rougeL_f1_score":0.1359575611,"nlg_score":0.0880621978}
{"Model Name":"Qwen3-30B-A3B","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"30500000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.1212751301,"summarization_PnSummary_rougeL_recall":0.3923323141,"summarization_PnSummary_rougeL_f1_score":0.1804727387,"nlg_score":0.164118288}
{"Model Name":"Llama-3.2-3B-Instruct","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"3210000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.151465786,"summarization_PnSummary_rougeL_recall":0.3775823327,"summarization_PnSummary_rougeL_f1_score":0.203395452,"nlg_score":0.1129755187}
{"Model Name":"Llama-3.2-1B-Instruct","thinking_method":"❌","model_url":"https:\/\/google.com","parameters_count":"1240000000","source_type":"Open-Source","summarization_PnSummary_rougeL_precision":0.2061378815,"summarization_PnSummary_rougeL_recall":0.2755376589,"summarization_PnSummary_rougeL_f1_score":0.2192316506,"nlg_score":0.0823387318}
{"Model Name":"gemini-2.5-pro","thinking_method":"✔️","model_url":"https:\/\/google.com","parameters_count":"None","source_type":"Closed-Source","summarization_PnSummary_rougeL_precision":0.1091020199,"summarization_PnSummary_rougeL_recall":0.3487472682,"summarization_PnSummary_rougeL_f1_score":0.1614333679,"nlg_score":0.1151518212}