diff --git "a/scores/gme-Qwen2-VL-2B-Instruct.json" "b/scores/gme-Qwen2-VL-2B-Instruct.json" --- "a/scores/gme-Qwen2-VL-2B-Instruct.json" +++ "b/scores/gme-Qwen2-VL-2B-Instruct.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "gme-Qwen2-VL-2B-Instruct", - "report_generated_date": "2025-06-09T07:04:30.518891", - "model_size": 2.21, - "url": "https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.007017" }, "metrics": { "image": { @@ -1017,765 +1014,287 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.774, - "hit@5": 0.878, - "hit@10": 0.91, - "ndcg_linear@1": 0.774, - "ndcg_linear@5": 0.8276032117095806, - "ndcg_linear@10": 0.8379559054452919, - "ndcg_exponential@1": 0.774, - "ndcg_exponential@5": 0.8276032117095806, - "ndcg_exponential@10": 0.8379559054452919, - "precision@1": 0.774, - "precision@5": 0.1756, - "precision@10": 0.091, - "recall@1": 0.774, - "recall@5": 0.878, - "recall@10": 0.91, - "f1@1": 0.774, - "f1@5": 0.2926666666666667, - "f1@10": 0.16545454545454544, - "map@1": 0.774, - "map@5": 0.811, - "map@10": 0.8152769841269842, - "mrr@1": 0.774, - "mrr@5": 0.811, - "mrr@10": 0.8152769841269842, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.4434589800443459, - "hit@5": 0.6075388026607539, - "hit@10": 0.6651884700665188, - "ndcg_linear@1": 0.4434589800443459, - "ndcg_linear@5": 0.5311135724096901, - "ndcg_linear@10": 0.549126558201007, - "ndcg_exponential@1": 0.4434589800443459, - "ndcg_exponential@5": 0.5311135724096901, - "ndcg_exponential@10": 0.549126558201007, - "precision@1": 0.4434589800443459, - "precision@5": 0.12416851441241683, - "precision@10": 0.06829268292682926, - "recall@1": 0.437250554323725, - "recall@5": 0.6023281596452328, - "recall@10": 0.6581116038433111, - "f1@1": 0.43902439024390244, - "f1@5": 0.20455818691112812, - "f1@10": 0.12296562762194693, - "map@1": 0.4434589800443459, - "map@5": 0.505469327420547, - "map@10": 0.512853601344455, - "mrr@1": 0.4434589800443459, - "mrr@5": 0.5099408721359941, - "mrr@10": 0.5174251223031711, - "num_pred": 451, - "num_data": 451 - }, - "ViDoRe_infovqa": { - "hit@1": 0.8481781376518218, - "hit@5": 0.9453441295546559, - "hit@10": 0.9615384615384616, - "ndcg_linear@1": 0.8481781376518218, - "ndcg_linear@5": 0.9016308878511893, - "ndcg_linear@10": 0.9073181214772046, - "ndcg_exponential@1": 0.8481781376518218, - "ndcg_exponential@5": 0.9016308878511893, - "ndcg_exponential@10": 0.9073181214772046, - "precision@1": 0.8481781376518218, - "precision@5": 0.1890688259109312, - "precision@10": 0.09635627530364373, - "recall@1": 0.8481781376518218, - "recall@5": 0.9427125506072874, - "recall@10": 0.9599190283400809, - "f1@1": 0.8481781376518218, - "f1@5": 0.3147484094852516, - "f1@10": 0.17503373819163293, - "map@1": 0.8481781376518218, - "map@5": 0.8873481781376519, - "map@10": 0.8898359681254419, - "mrr@1": 0.8481781376518218, - "mrr@5": 0.8883603238866398, - "mrr@10": 0.8905589293747189, - "num_pred": 494, - "num_data": 494 - }, - "ViDoRe_tabfquad": { - "hit@1": 0.8892857142857142, - "hit@5": 0.9678571428571429, - "hit@10": 0.975, - "ndcg_linear@1": 0.8892857142857142, - "ndcg_linear@5": 0.9332834391651387, - "ndcg_linear@10": 0.9356005756298494, - "ndcg_exponential@1": 0.8892857142857142, - "ndcg_exponential@5": 0.9332834391651387, - "ndcg_exponential@10": 0.9356005756298494, - "precision@1": 0.8892857142857142, - "precision@5": 0.19357142857142853, - "precision@10": 0.09749999999999999, - "recall@1": 0.8892857142857142, - "recall@5": 0.9678571428571429, - "recall@10": 0.975, - "f1@1": 0.8892857142857142, - "f1@5": 0.3226190476190477, - "f1@10": 0.1772727272727273, - "map@1": 0.8892857142857142, - "map@5": 0.921547619047619, - "map@10": 0.9225042517006803, - "mrr@1": 0.8892857142857142, - "mrr@5": 0.921547619047619, - "mrr@10": 0.9225042517006803, - "num_pred": 280, - "num_data": 280 - }, - "ViDoRe_tatdqa": { - "hit@1": 0.5577156743620899, - "hit@5": 0.8250303766707169, - "hit@10": 0.8894289185905225, - "ndcg_linear@1": 0.5577156743620899, - "ndcg_linear@5": 0.6993890766047229, - "ndcg_linear@10": 0.7206656381690775, - "ndcg_exponential@1": 0.5577156743620899, - "ndcg_exponential@5": 0.6993890766047229, - "ndcg_exponential@10": 0.7206656381690775, - "precision@1": 0.5577156743620899, - "precision@5": 0.16537059538274604, - "precision@10": 0.08924665856622113, - "recall@1": 0.5562575941676793, - "recall@5": 0.8235722964763061, - "recall@10": 0.8885783718104495, - "f1@1": 0.556611068154203, - "f1@5": 0.2752357808250883, - "f1@10": 0.1621156890901727, - "map@1": 0.5577156743620899, - "map@5": 0.6577612393681652, - "map@10": 0.6667486354606647, - "mrr@1": 0.5577156743620899, - "mrr@5": 0.6586776022681248, - "mrr@10": 0.6675377056452391, - "num_pred": 1646, - "num_data": 1646 - }, - "ViDoRe_shiftproject": { - "hit@1": 0.8, - "hit@5": 0.97, - "hit@10": 0.99, - "ndcg_linear@1": 0.8, - "ndcg_linear@5": 0.894703974563633, - "ndcg_linear@10": 0.9010476078536062, - "ndcg_exponential@1": 0.8, - "ndcg_exponential@5": 0.894703974563633, - "ndcg_exponential@10": 0.9010476078536062, - "precision@1": 0.8, - "precision@5": 0.19399999999999998, - "precision@10": 0.09899999999999999, - "recall@1": 0.8, - "recall@5": 0.97, - "recall@10": 0.99, - "f1@1": 0.8, - "f1@5": 0.3233333333333334, - "f1@10": 0.18000000000000008, - "map@1": 0.8, - "map@5": 0.8691666666666665, - "map@10": 0.8717063492063492, - "mrr@1": 0.8, - "mrr@5": 0.8691666666666665, - "mrr@10": 0.8717063492063492, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.96, - "hit@5": 0.99, - "hit@10": 1.0, - "ndcg_linear@1": 0.96, - "ndcg_linear@5": 0.97517782560806, - "ndcg_linear@10": 0.9787398974791403, - "ndcg_exponential@1": 0.96, - "ndcg_exponential@5": 0.97517782560806, - "ndcg_exponential@10": 0.9787398974791403, - "precision@1": 0.96, - "precision@5": 0.19799999999999998, - "precision@10": 0.09999999999999998, - "recall@1": 0.96, - "recall@5": 0.99, - "recall@10": 1.0, - "f1@1": 0.96, - "f1@5": 0.3300000000000001, - "f1@10": 0.1818181818181819, - "map@1": 0.96, - "map@5": 0.9703333333333333, - "map@10": 0.972, - "mrr@1": 0.96, - "mrr@5": 0.9703333333333333, - "mrr@10": 0.972, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.88, - "hit@5": 0.95, - "hit@10": 0.96, - "ndcg_linear@1": 0.88, - "ndcg_linear@5": 0.9191057182152036, - "ndcg_linear@10": 0.9221160181718434, - "ndcg_exponential@1": 0.88, - "ndcg_exponential@5": 0.9191057182152036, - "ndcg_exponential@10": 0.9221160181718434, - "precision@1": 0.88, - "precision@5": 0.18999999999999997, - "precision@10": 0.09599999999999997, - "recall@1": 0.88, - "recall@5": 0.95, - "recall@10": 0.96, - "f1@1": 0.88, - "f1@5": 0.3166666666666667, - "f1@10": 0.17454545454545464, - "map@1": 0.88, - "map@5": 0.9086666666666667, - "map@10": 0.9097777777777778, - "mrr@1": 0.88, - "mrr@5": 0.9086666666666667, - "mrr@10": 0.9097777777777778, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.91, - "hit@5": 0.97, - "hit@10": 0.98, - "ndcg_linear@1": 0.91, - "ndcg_linear@5": 0.9458532532593068, - "ndcg_linear@10": 0.9487439015224857, - "ndcg_exponential@1": 0.91, - "ndcg_exponential@5": 0.9458532532593068, - "ndcg_exponential@10": 0.9487439015224857, - "precision@1": 0.91, - "precision@5": 0.19399999999999995, - "precision@10": 0.09799999999999998, - "recall@1": 0.91, - "recall@5": 0.97, - "recall@10": 0.98, - "f1@1": 0.91, - "f1@5": 0.3233333333333334, - "f1@10": 0.17818181818181827, - "map@1": 0.91, - "map@5": 0.9375, - "map@10": 0.9384999999999999, - "mrr@1": 0.91, - "mrr@5": 0.9375, - "mrr@10": 0.9384999999999999, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.97, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.97, - "ndcg_linear@5": 0.9869253606521631, - "ndcg_linear@10": 0.9869253606521631, - "ndcg_exponential@1": 0.97, - "ndcg_exponential@5": 0.9869253606521631, - "ndcg_exponential@10": 0.9869253606521631, - "precision@1": 0.97, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.97, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.97, - "f1@5": 0.3333333333333335, - "f1@10": 0.1818181818181819, - "map@1": 0.97, - "map@5": 0.9825, - "map@10": 0.9825, - "mrr@1": 0.97, - "mrr@5": 0.9825, - "mrr@10": 0.9825, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.6153846153846154, - "hit@5": 0.7884615384615384, - "hit@10": 0.8653846153846154, - "ndcg_linear@1": 0.5897435897435896, - "ndcg_linear@5": 0.6094691508995544, - "ndcg_linear@10": 0.635637750038373, - "ndcg_exponential@1": 0.5824175824175823, - "ndcg_exponential@5": 0.6103242475099454, - "ndcg_exponential@10": 0.6356245865615132, - "precision@1": 0.6153846153846154, - "precision@5": 0.23461538461538461, - "precision@10": 0.1423076923076923, - "recall@1": 0.4133241758241758, - "recall@5": 0.6478021978021977, - "recall@10": 0.7252372627372627, - "f1@1": 0.4552021696252465, - "f1@5": 0.3082283729342553, - "f1@10": 0.2158892483835923, - "map@1": 0.6153846153846154, - "map@5": 0.5464262820512821, - "map@10": 0.5583285002035001, - "mrr@1": 0.6153846153846154, - "mrr@5": 0.6769230769230768, - "mrr@10": 0.6880036630036629, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.54375, - "hit@5": 0.8375, - "hit@10": 0.88125, - "ndcg_linear@1": 0.54375, - "ndcg_linear@5": 0.5823464823989074, - "ndcg_linear@10": 0.606062263411702, - "ndcg_exponential@1": 0.54375, - "ndcg_exponential@5": 0.5823464823989074, - "ndcg_exponential@10": 0.606062263411702, - "precision@1": 0.54375, - "precision@5": 0.26625000000000004, - "precision@10": 0.16687500000000002, - "recall@1": 0.34037779316271966, - "recall@5": 0.6160034695696461, - "recall@10": 0.6953493840718105, - "f1@1": 0.3838277867965368, - "f1@5": 0.32790766872016874, - "f1@10": 0.24135968266482974, - "map@1": 0.54375, - "map@5": 0.5093559027777778, - "map@10": 0.5200279313114134, - "mrr@1": 0.54375, - "mrr@5": 0.6529166666666667, - "mrr@10": 0.6586954365079365, - "num_pred": 160, - "num_data": 160 - }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.490625, - "hit@5": 0.796875, - "hit@10": 0.86875, - "ndcg_linear@1": 0.490625, - "ndcg_linear@5": 0.5397544908352085, - "ndcg_linear@10": 0.5693203555054913, - "ndcg_exponential@1": 0.490625, - "ndcg_exponential@5": 0.5397544908352085, - "ndcg_exponential@10": 0.5693203555054912, - "precision@1": 0.490625, - "precision@5": 0.251875, - "precision@10": 0.16140625000000003, - "recall@1": 0.2978885162019353, - "recall@5": 0.5770897790934556, - "recall@10": 0.6719354787369494, - "f1@1": 0.3395271915584416, - "f1@5": 0.3084593392718393, - "f1@10": 0.23304521375615125, - "map@1": 0.490625, - "map@5": 0.46664019097222215, - "map@10": 0.479486573188303, - "mrr@1": 0.490625, - "mrr@5": 0.6060416666666667, - "mrr@10": 0.6155078125, - "num_pred": 640, - "num_data": 640 - }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.603448275862069, - "hit@5": 0.8793103448275862, - "hit@10": 0.9137931034482759, - "ndcg_linear@1": 0.603448275862069, - "ndcg_linear@5": 0.5903068962042065, - "ndcg_linear@10": 0.5492779808098573, - "ndcg_exponential@1": 0.603448275862069, - "ndcg_exponential@5": 0.5903068962042065, - "ndcg_exponential@10": 0.5492779808098572, - "precision@1": 0.603448275862069, - "precision@5": 0.5206896551724138, - "precision@10": 0.39999999999999997, - "recall@1": 0.07380447356267512, - "recall@5": 0.3068313928663318, - "recall@10": 0.41764795564447366, - "f1@1": 0.11461138038552825, - "f1@5": 0.3138737513023153, - "f1@10": 0.3342336255680905, - "map@1": 0.603448275862069, - "map@5": 0.504985632183908, - "map@10": 0.4221401747161189, - "mrr@1": 0.603448275862069, - "mrr@5": 0.718103448275862, - "mrr@10": 0.7221743295019157, - "num_pred": 58, - "num_data": 58 - }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.5689655172413793, - "hit@5": 0.8491379310344828, - "hit@10": 0.8879310344827587, - "ndcg_linear@1": 0.5689655172413793, - "ndcg_linear@5": 0.5019343305838408, - "ndcg_linear@10": 0.47979676122532555, - "ndcg_exponential@1": 0.5689655172413793, - "ndcg_exponential@5": 0.5019343305838408, - "ndcg_exponential@10": 0.47979676122532555, - "precision@1": 0.5689655172413793, - "precision@5": 0.428448275862069, - "precision@10": 0.34698275862068967, - "recall@1": 0.08034274543141959, - "recall@5": 0.25237595480423564, - "recall@10": 0.3645843669588603, - "f1@1": 0.11905920072661083, - "f1@5": 0.25183497028873714, - "f1@10": 0.2856798311584853, - "map@1": 0.5689655172413793, - "map@5": 0.40593749999999995, - "map@10": 0.35002105419913815, - "mrr@1": 0.5689655172413793, - "mrr@5": 0.6777298850574713, - "mrr@10": 0.682835591133005, - "num_pred": 232, - "num_data": 232 - }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.5087719298245614, - "hit@5": 0.8421052631578947, - "hit@10": 0.9298245614035088, - "ndcg_linear@1": 0.5087719298245614, - "ndcg_linear@5": 0.5592665259548532, - "ndcg_linear@10": 0.5942776109073339, - "ndcg_exponential@1": 0.5087719298245614, - "ndcg_exponential@5": 0.5592665259548532, - "ndcg_exponential@10": 0.5942776109073339, - "precision@1": 0.5087719298245614, - "precision@5": 0.2701754385964912, - "precision@10": 0.17894736842105266, - "recall@1": 0.2888888888888889, - "recall@5": 0.6026733500417711, - "recall@10": 0.7070384294068504, - "f1@1": 0.3382218171691856, - "f1@5": 0.3262657623979285, - "f1@10": 0.2492540181802036, - "map@1": 0.5087719298245614, - "map@5": 0.4741812865497076, - "map@10": 0.4867362851573378, - "mrr@1": 0.5087719298245614, - "mrr@5": 0.6342105263157894, - "mrr@10": 0.6484126984126984, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.4473684210526316, - "hit@5": 0.793859649122807, - "hit@10": 0.8771929824561403, - "ndcg_linear@1": 0.4473684210526316, - "ndcg_linear@5": 0.5073413437187128, - "ndcg_linear@10": 0.5436696652694063, - "ndcg_exponential@1": 0.4473684210526316, - "ndcg_exponential@5": 0.5073413437187128, - "ndcg_exponential@10": 0.5436696652694063, - "precision@1": 0.4473684210526316, - "precision@5": 0.2429824561403509, - "precision@10": 0.1635964912280702, - "recall@1": 0.2503968253968254, - "recall@5": 0.5527621136173767, - "recall@10": 0.6586466165413534, - "f1@1": 0.29117386075643786, - "f1@5": 0.2958373843475797, - "f1@10": 0.22844515727846343, - "map@1": 0.4473684210526316, - "map@5": 0.4241118421052631, - "map@10": 0.4392249141371949, - "mrr@1": 0.4473684210526316, - "mrr@5": 0.5828216374269005, - "mrr@10": 0.5950205374547479, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.7610294117647058, - "hit@5": 0.8713235294117647, - "hit@10": 0.9068627450980392, - "ndcg_linear@1": 0.7610294117647058, - "ndcg_linear@5": 0.8199677874725917, - "ndcg_linear@10": 0.8315054893421978, - "ndcg_exponential@1": 0.7610294117647058, - "ndcg_exponential@5": 0.8199677874725917, - "ndcg_exponential@10": 0.8315054893421978, - "precision@1": 0.7610294117647058, - "precision@5": 0.17426470588235293, - "precision@10": 0.09068627450980392, - "recall@1": 0.7610294117647058, - "recall@5": 0.8713235294117647, - "recall@10": 0.9068627450980392, - "f1@1": 0.7610294117647058, - "f1@5": 0.2904411764705883, - "f1@10": 0.1648841354723708, - "map@1": 0.7610294117647058, - "map@5": 0.8028799019607843, - "map@10": 0.8076700124494243, - "mrr@1": 0.7610294117647058, - "mrr@5": 0.8028799019607843, - "mrr@10": 0.8076700124494243, - "num_pred": 816, - "num_data": 816 + "video": { + "K700": { + "hit@1": 0.352, + "hit@5": 0.558, + "hit@10": 0.634, + "ndcg_linear@1": 0.352, + "ndcg_linear@5": 0.4624292100503551, + "ndcg_linear@10": 0.4871702819761314, + "ndcg_exponential@1": 0.352, + "ndcg_exponential@5": 0.4624292100503551, + "ndcg_exponential@10": 0.4871702819761314, + "precision@1": 0.352, + "precision@5": 0.11159999999999999, + "precision@10": 0.06339999999999998, + "recall@1": 0.352, + "recall@5": 0.558, + "recall@10": 0.634, + "f1@1": 0.352, + "f1@5": 0.18600000000000005, + "f1@10": 0.11527272727272728, + "map@1": 0.352, + "map@5": 0.4306, + "map@10": 0.440915873015873, + "mrr@1": 0.352, + "mrr@5": 0.4306, + "mrr@10": 0.440915873015873, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_ChartQA": { - "hit@1": 0.6825396825396826, - "hit@5": 0.9047619047619048, - "hit@10": 0.9206349206349206, - "ndcg_linear@1": 0.6825396825396826, - "ndcg_linear@5": 0.7988138511124628, - "ndcg_linear@10": 0.8034021816889372, - "ndcg_exponential@1": 0.6825396825396826, - "ndcg_exponential@5": 0.7988138511124628, - "ndcg_exponential@10": 0.8034021816889372, - "precision@1": 0.6825396825396826, - "precision@5": 0.18095238095238086, - "precision@10": 0.09206349206349201, - "recall@1": 0.6825396825396826, - "recall@5": 0.9047619047619048, - "recall@10": 0.9206349206349206, - "f1@1": 0.6825396825396826, - "f1@5": 0.3015873015873015, - "f1@10": 0.16738816738816742, - "map@1": 0.6825396825396826, - "map@5": 0.7640211640211642, - "map@10": 0.7656084656084656, - "mrr@1": 0.6825396825396826, - "mrr@5": 0.7640211640211642, - "mrr@10": 0.7656084656084656, - "num_pred": 63, - "num_data": 63 + "SmthSmthV2": { + "hit@1": 0.299, + "hit@5": 0.528, + "hit@10": 0.622, + "ndcg_linear@1": 0.299, + "ndcg_linear@5": 0.41899491853941084, + "ndcg_linear@10": 0.44961784439388147, + "ndcg_exponential@1": 0.299, + "ndcg_exponential@5": 0.41899491853941084, + "ndcg_exponential@10": 0.44961784439388147, + "precision@1": 0.299, + "precision@5": 0.1056, + "precision@10": 0.0622, + "recall@1": 0.299, + "recall@5": 0.528, + "recall@10": 0.622, + "f1@1": 0.299, + "f1@5": 0.17600000000000005, + "f1@10": 0.11309090909090909, + "map@1": 0.299, + "map@5": 0.38288333333333335, + "map@10": 0.3956539682539683, + "mrr@1": 0.299, + "mrr@5": 0.38288333333333335, + "mrr@10": 0.3956539682539683, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.7648054145516074, - "hit@5": 0.9035532994923858, - "hit@10": 0.937394247038917, - "ndcg_linear@1": 0.7648054145516074, - "ndcg_linear@5": 0.8442477575170283, - "ndcg_linear@10": 0.8553129247371343, - "ndcg_exponential@1": 0.7648054145516074, - "ndcg_exponential@5": 0.8442477575170283, - "ndcg_exponential@10": 0.8553129247371343, - "precision@1": 0.7648054145516074, - "precision@5": 0.18071065989847712, - "precision@10": 0.0937394247038917, - "recall@1": 0.7648054145516074, - "recall@5": 0.9035532994923858, - "recall@10": 0.937394247038917, - "f1@1": 0.7648054145516074, - "f1@5": 0.3011844331641287, - "f1@10": 0.1704353176434395, - "map@1": 0.7648054145516074, - "map@5": 0.8239988719684151, - "map@10": 0.8286332017296485, - "mrr@1": 0.7648054145516074, - "mrr@5": 0.8239988719684151, - "mrr@10": 0.8286332017296485, - "num_pred": 591, - "num_data": 591 + "HMDB51": { + "hit@1": 0.434, + "hit@5": 0.672, + "hit@10": 0.768, + "ndcg_linear@1": 0.434, + "ndcg_linear@5": 0.5564379559041219, + "ndcg_linear@10": 0.587723216959859, + "ndcg_exponential@1": 0.434, + "ndcg_exponential@5": 0.5564379559041219, + "ndcg_exponential@10": 0.587723216959859, + "precision@1": 0.434, + "precision@5": 0.13439999999999996, + "precision@10": 0.0768, + "recall@1": 0.434, + "recall@5": 0.672, + "recall@10": 0.768, + "f1@1": 0.434, + "f1@5": 0.22400000000000006, + "f1@10": 0.13963636363636361, + "map@1": 0.434, + "map@5": 0.5183166666666666, + "map@10": 0.531368253968254, + "mrr@1": 0.434, + "mrr@5": 0.5183166666666666, + "mrr@10": 0.531368253968254, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_SlideVQA": { - "hit@1": 0.9064748201438849, - "hit@5": 0.9802158273381295, - "hit@10": 0.9856115107913669, - "ndcg_linear@1": 0.9064748201438849, - "ndcg_linear@5": 0.9337625442422413, - "ndcg_linear@10": 0.9378099940259211, - "ndcg_exponential@1": 0.9064748201438849, - "ndcg_exponential@5": 0.9337625442422413, - "ndcg_exponential@10": 0.9378099940259211, - "precision@1": 0.9064748201438849, - "precision@5": 0.24244604316546764, - "precision@10": 0.12302158273381293, - "recall@1": 0.783273381294964, - "recall@5": 0.966726618705036, - "recall@10": 0.9775179856115108, - "f1@1": 0.8243405275779376, - "f1@5": 0.380695443645084, - "f1@10": 0.21601809461521695, - "map@1": 0.9064748201438849, - "map@5": 0.9146432853717026, - "map@10": 0.9169378782688133, - "mrr@1": 0.9064748201438849, - "mrr@5": 0.9376199040767385, - "mrr@10": 0.938401421719767, - "num_pred": 556, - "num_data": 556 + "UCF101": { + "hit@1": 0.524, + "hit@5": 0.745, + "hit@10": 0.821, + "ndcg_linear@1": 0.524, + "ndcg_linear@5": 0.6452214501789469, + "ndcg_linear@10": 0.6701386847278763, + "ndcg_exponential@1": 0.524, + "ndcg_exponential@5": 0.6452214501789469, + "ndcg_exponential@10": 0.6701386847278763, + "precision@1": 0.524, + "precision@5": 0.149, + "precision@10": 0.0821, + "recall@1": 0.524, + "recall@5": 0.745, + "recall@10": 0.821, + "f1@1": 0.524, + "f1@5": 0.2483333333333334, + "f1@10": 0.14927272727272728, + "map@1": 0.524, + "map@5": 0.6116833333333334, + "map@10": 0.6221575396825396, + "mrr@1": 0.524, + "mrr@5": 0.6116833333333334, + "mrr@10": 0.6221575396825396, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_InfoVQA": { - "hit@1": 0.8426183844011143, - "hit@5": 0.9651810584958217, - "hit@10": 0.9874651810584958, - "ndcg_linear@1": 0.8426183844011143, - "ndcg_linear@5": 0.9135602456956691, - "ndcg_linear@10": 0.9210168523969697, - "ndcg_exponential@1": 0.8426183844011143, - "ndcg_exponential@5": 0.9135602456956691, - "ndcg_exponential@10": 0.9210168523969697, - "precision@1": 0.8426183844011143, - "precision@5": 0.1930362116991643, - "precision@10": 0.09874651810584957, - "recall@1": 0.8426183844011143, - "recall@5": 0.9651810584958217, - "recall@10": 0.9874651810584958, - "f1@1": 0.8426183844011143, - "f1@5": 0.3217270194986073, - "f1@10": 0.17953912382881745, - "map@1": 0.8426183844011143, - "map@5": 0.8958681522748375, - "map@10": 0.8990941548392801, - "mrr@1": 0.8426183844011143, - "mrr@5": 0.8958681522748375, - "mrr@10": 0.8990941548392801, - "num_pred": 718, - "num_data": 718 + "Breakfast": { + "hit@1": 0.13625866050808313, + "hit@5": 0.5935334872979214, + "hit@10": 1.0, + "ndcg_linear@1": 0.13625866050808313, + "ndcg_linear@5": 0.3556714764778745, + "ndcg_linear@10": 0.4859126067666473, + "ndcg_exponential@1": 0.13625866050808313, + "ndcg_exponential@5": 0.3556714764778745, + "ndcg_exponential@10": 0.4859126067666473, + "precision@1": 0.13625866050808313, + "precision@5": 0.11870669745958429, + "precision@10": 0.1, + "recall@1": 0.13625866050808313, + "recall@5": 0.5935334872979214, + "recall@10": 1.0, + "f1@1": 0.13625866050808313, + "f1@5": 0.19784449576597388, + "f1@10": 0.1818181818181818, + "map@1": 0.13625866050808313, + "map@5": 0.27886836027713624, + "map@10": 0.3318990798782947, + "mrr@1": 0.13625866050808313, + "mrr@5": 0.27886836027713624, + "mrr@10": 0.3318990798782947, + "num_pred": 433, + "num_data": 433 }, - "VisRAG_PlotQA": { - "hit@1": 0.5052143684820394, - "hit@5": 0.764774044032445, - "hit@10": 0.8551564310544612, - "ndcg_linear@1": 0.5052143684820394, - "ndcg_linear@5": 0.640903570079321, - "ndcg_linear@10": 0.6707054604251023, - "ndcg_exponential@1": 0.5052143684820394, - "ndcg_exponential@5": 0.640903570079321, - "ndcg_exponential@10": 0.6707054604251023, - "precision@1": 0.5052143684820394, - "precision@5": 0.15295480880648896, - "precision@10": 0.0855156431054461, - "recall@1": 0.5052143684820394, - "recall@5": 0.764774044032445, - "recall@10": 0.8551564310544612, - "f1@1": 0.5052143684820394, - "f1@5": 0.25492468134414836, - "f1@10": 0.15548298746444753, - "map@1": 0.5052143684820394, - "map@5": 0.5998648126689843, - "map@10": 0.612502069193842, - "mrr@1": 0.5052143684820394, - "mrr@5": 0.5998648126689843, - "mrr@10": 0.612502069193842, - "num_pred": 863, - "num_data": 863 + "MVBench": { + "hit@1": 0.3745, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3745, + "ndcg_linear@5": 0.7155447146201682, + "ndcg_linear@10": 0.7155447146201682, + "ndcg_exponential@1": 0.3745, + "ndcg_exponential@5": 0.7155447146201682, + "ndcg_exponential@10": 0.7155447146201682, + "precision@1": 0.3745, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.3745, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3745, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.3745, + "map@5": 0.6196291666666666, + "map@10": 0.6196291666666666, + "mrr@1": 0.3745, + "mrr@5": 0.6196291666666666, + "mrr@10": 0.6196291666666666, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoSeek-page": { - "hit@1": 0.01838879159369527, - "hit@5": 0.4089316987740806, - "hit@10": 0.6401050788091068, - "ndcg_linear@1": 0.01838879159369527, - "ndcg_linear@5": 0.21620008992198805, - "ndcg_linear@10": 0.2907931583080436, - "ndcg_exponential@1": 0.01838879159369527, - "ndcg_exponential@5": 0.21620008992198805, - "ndcg_exponential@10": 0.2907931583080436, - "precision@1": 0.01838879159369527, - "precision@5": 0.08178633975481611, - "precision@10": 0.06401050788091067, - "recall@1": 0.01838879159369527, - "recall@5": 0.4089316987740806, - "recall@10": 0.6401050788091068, - "f1@1": 0.01838879159369527, - "f1@5": 0.13631056625802687, - "f1@10": 0.1163827416016558, - "map@1": 0.01838879159369527, - "map@5": 0.15302101576182137, - "map@10": 0.1837190392794596, - "mrr@1": 0.01838879159369527, - "mrr@5": 0.15302101576182137, - "mrr@10": 0.1837190392794596, - "num_pred": 1142, - "num_data": 1142 + "Video-MME": { + "hit@1": 0.3425925925925926, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3425925925925926, + "ndcg_linear@5": 0.6913650713075382, + "ndcg_linear@10": 0.6913650713075382, + "ndcg_exponential@1": 0.3425925925925926, + "ndcg_exponential@5": 0.6913650713075382, + "ndcg_exponential@10": 0.6913650713075382, + "precision@1": 0.3425925925925926, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.3425925925925926, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3425925925925926, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.3425925925925926, + "map@5": 0.5881790123456789, + "map@10": 0.5881790123456789, + "mrr@1": 0.3425925925925926, + "mrr@5": 0.5881790123456789, + "mrr@10": 0.5881790123456789, + "num_pred": 2700, + "num_data": 2700 }, - "ViDoSeek-doc": { - "hit@1": 0.9964973730297724, - "hit@5": 0.9982486865148862, - "hit@10": 0.9982486865148862, - "ndcg_linear@1": 0.678342089900759, - "ndcg_linear@5": 0.8362374075645319, - "ndcg_linear@10": 0.8165935940149865, - "ndcg_exponential@1": 0.4510883162371779, - "ndcg_exponential@5": 0.7145068150665362, - "ndcg_exponential@10": 0.7410897811756266, - "precision@1": 0.9964973730297724, - "precision@5": 0.9262697022767075, - "precision@10": 0.8286339754816112, - "recall@1": 0.05876920449269009, - "recall@5": 0.2726143296172191, - "recall@10": 0.4766810037927378, - "f1@1": 0.11024247532054007, - "f1@5": 0.4126599828359686, - "f1@10": 0.5917511632990256, - "map@1": 0.9964973730297724, - "map@5": 0.9193257443082313, - "map@10": 0.8188673981787047, - "mrr@1": 0.9964973730297724, - "mrr@5": 0.997008172796264, - "mrr@10": 0.997008172796264, - "num_pred": 1142, - "num_data": 1142 + "NExTQA": { + "hit@1": 0.3949089210649229, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.3949089210649229, + "ndcg_linear@5": 0.7033624022532277, + "ndcg_linear@10": 0.7033624022532277, + "ndcg_exponential@1": 0.3949089210649229, + "ndcg_exponential@5": 0.7033624022532277, + "ndcg_exponential@10": 0.7033624022532277, + "precision@1": 0.3949089210649229, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.3949089210649229, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.3949089210649229, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.3949089210649229, + "map@5": 0.605927915304375, + "map@10": 0.605927915304375, + "mrr@1": 0.3949089210649229, + "mrr@5": 0.6057527635061498, + "mrr@10": 0.6057527635061498, + "num_pred": 8564, + "num_data": 8564 }, - "MMLongBench-page": { - "hit@1": 0.07637231503579953, - "hit@5": 0.3031026252983294, - "hit@10": 0.4033412887828162, - "ndcg_linear@1": 0.07637231503579953, - "ndcg_linear@5": 0.15820239639498082, - "ndcg_linear@10": 0.19395841349578616, - "ndcg_exponential@1": 0.07637231503579953, - "ndcg_exponential@5": 0.15820239639498082, - "ndcg_exponential@10": 0.19395841349578616, - "precision@1": 0.07637231503579953, - "precision@5": 0.07159904534606205, - "precision@10": 0.052983293556085914, - "recall@1": 0.048528241845664274, - "recall@5": 0.22104074326628023, - "recall@10": 0.32010313672008184, - "f1@1": 0.05683207007789346, - "f1@5": 0.10227087246601498, - "f1@10": 0.08682064083863344, - "map@1": 0.07637231503579953, - "map@5": 0.11957305754441794, - "map@10": 0.13619920114028108, - "mrr@1": 0.07637231503579953, - "mrr@5": 0.16328560063643593, - "mrr@10": 0.17665170284502024, - "num_pred": 838, - "num_data": 838 + "EgoSchema": { + "hit@1": 0.408, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.408, + "ndcg_linear@5": 0.7172123608985213, + "ndcg_linear@10": 0.7172123608985213, + "ndcg_exponential@1": 0.408, + "ndcg_exponential@5": 0.7172123608985213, + "ndcg_exponential@10": 0.7172123608985213, + "precision@1": 0.408, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.408, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.408, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.408, + "map@5": 0.6235, + "map@10": 0.6235, + "mrr@1": 0.408, + "mrr@5": 0.6235, + "mrr@10": 0.6235, + "num_pred": 500, + "num_data": 500 + }, + "ActivityNetQA": { + "hit@1": 0.58, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.58, + "ndcg_linear@5": 0.8449904965000122, + "ndcg_linear@10": 0.8449904965000122, + "ndcg_exponential@1": 0.58, + "ndcg_exponential@5": 0.8449904965000122, + "ndcg_exponential@10": 0.8449904965000122, + "precision@1": 0.58, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.58, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.58, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.58, + "map@5": 0.79, + "map@10": 0.79, + "mrr@1": 0.58, + "mrr@5": 0.79, + "mrr@10": 0.79, + "num_pred": 1000, + "num_data": 1000 }, - "MMLongBench-doc": { - "hit@1": 0.698090692124105, - "hit@5": 0.8007159904534606, - "hit@10": 0.8281622911694511, - "ndcg_linear@1": 0.49562450278440734, - "ndcg_linear@5": 0.5143162653023697, - "ndcg_linear@10": 0.49047862420126026, - "ndcg_exponential@1": 0.35100579611319466, - "ndcg_exponential@5": 0.43113924192770037, - "ndcg_exponential@10": 0.43753218599979815, - "precision@1": 0.698090692124105, - "precision@5": 0.5663484486873509, - "precision@10": 0.4947494033412887, - "recall@1": 0.022961848267359367, - "recall@5": 0.08932041189913144, - "recall@10": 0.15198617349694285, - "f1@1": 0.044005822895889966, - "f1@5": 0.1481301071634879, - "f1@10": 0.217939992851331, - "map@1": 0.698090692124105, - "map@5": 0.5353301511535402, - "map@10": 0.44876554785266, - "mrr@1": 0.698090692124105, - "mrr@5": 0.7382259347653143, - "mrr@10": 0.7417945221047847, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { "DiDeMo": { "hit@1": 0.2201195219123506, "hit@5": 0.41733067729083667, @@ -1804,481 +1323,875 @@ "num_pred": 1004, "num_data": 1004 }, - "MSR-VTT": { - "hit@1": 0.273, - "hit@5": 0.498, - "hit@10": 0.601, - "ndcg_linear@1": 0.273, - "ndcg_linear@5": 0.39500395524644993, - "ndcg_linear@10": 0.4288322570956004, - "ndcg_exponential@1": 0.273, - "ndcg_exponential@5": 0.39500395524644993, - "ndcg_exponential@10": 0.4288322570956004, - "precision@1": 0.273, - "precision@5": 0.0996, - "precision@10": 0.060099999999999994, - "recall@1": 0.273, - "recall@5": 0.498, - "recall@10": 0.601, - "f1@1": 0.273, - "f1@5": 0.16600000000000006, - "f1@10": 0.1092727272727273, - "map@1": 0.273, - "map@5": 0.3605333333333333, - "map@10": 0.3747972222222222, - "mrr@1": 0.273, - "mrr@5": 0.3605333333333333, - "mrr@10": 0.3747972222222222, - "num_pred": 1000, - "num_data": 1000 + "MSR-VTT": { + "hit@1": 0.273, + "hit@5": 0.498, + "hit@10": 0.601, + "ndcg_linear@1": 0.273, + "ndcg_linear@5": 0.39500395524644993, + "ndcg_linear@10": 0.4288322570956004, + "ndcg_exponential@1": 0.273, + "ndcg_exponential@5": 0.39500395524644993, + "ndcg_exponential@10": 0.4288322570956004, + "precision@1": 0.273, + "precision@5": 0.0996, + "precision@10": 0.060099999999999994, + "recall@1": 0.273, + "recall@5": 0.498, + "recall@10": 0.601, + "f1@1": 0.273, + "f1@5": 0.16600000000000006, + "f1@10": 0.1092727272727273, + "map@1": 0.273, + "map@5": 0.3605333333333333, + "map@10": 0.3747972222222222, + "mrr@1": 0.273, + "mrr@5": 0.3605333333333333, + "mrr@10": 0.3747972222222222, + "num_pred": 1000, + "num_data": 1000 + }, + "MSVD": { + "hit@1": 0.4761194029850746, + "hit@5": 0.753731343283582, + "hit@10": 0.8223880597014925, + "ndcg_linear@1": 0.4761194029850746, + "ndcg_linear@5": 0.6277485828345324, + "ndcg_linear@10": 0.6502611190764599, + "ndcg_exponential@1": 0.4761194029850746, + "ndcg_exponential@5": 0.6277485828345324, + "ndcg_exponential@10": 0.6502611190764599, + "precision@1": 0.4761194029850746, + "precision@5": 0.15074626865671642, + "precision@10": 0.08223880597014925, + "recall@1": 0.4761194029850746, + "recall@5": 0.753731343283582, + "recall@10": 0.8223880597014925, + "f1@1": 0.4761194029850746, + "f1@5": 0.25124378109452744, + "f1@10": 0.14952510176390776, + "map@1": 0.4761194029850746, + "map@5": 0.5854726368159203, + "map@10": 0.5949484719260838, + "mrr@1": 0.4761194029850746, + "mrr@5": 0.5854726368159203, + "mrr@10": 0.5949484719260838, + "num_pred": 670, + "num_data": 670 + }, + "VATEX": { + "hit@1": 0.22956677087985708, + "hit@5": 0.44796784278695845, + "hit@10": 0.5482358195623046, + "ndcg_linear@1": 0.22956677087985708, + "ndcg_linear@5": 0.3442450550592858, + "ndcg_linear@10": 0.37658573093046976, + "ndcg_exponential@1": 0.22956677087985708, + "ndcg_exponential@5": 0.3442450550592858, + "ndcg_exponential@10": 0.37658573093046976, + "precision@1": 0.22956677087985708, + "precision@5": 0.0895935685573917, + "precision@10": 0.05482358195623046, + "recall@1": 0.22956677087985708, + "recall@5": 0.44796784278695845, + "recall@10": 0.5482358195623046, + "f1@1": 0.22956677087985708, + "f1@5": 0.1493226142623195, + "f1@10": 0.09967923992041902, + "map@1": 0.22956677087985708, + "map@5": 0.3098518683936281, + "map@10": 0.3231518109700334, + "mrr@1": 0.22956677087985708, + "mrr@5": 0.3098518683936281, + "mrr@10": 0.3231518109700334, + "num_pred": 4478, + "num_data": 4478 + }, + "YouCook2": { + "hit@1": 0.07895564642969487, + "hit@5": 0.18402013211701793, + "hit@10": 0.2491349480968858, + "ndcg_linear@1": 0.07895564642969487, + "ndcg_linear@5": 0.13201194607861527, + "ndcg_linear@10": 0.1529921311761448, + "ndcg_exponential@1": 0.07895564642969487, + "ndcg_exponential@5": 0.13201194607861527, + "ndcg_exponential@10": 0.1529921311761448, + "precision@1": 0.07895564642969487, + "precision@5": 0.03680402642340359, + "precision@10": 0.02491349480968858, + "recall@1": 0.07895564642969487, + "recall@5": 0.18402013211701793, + "recall@10": 0.2491349480968858, + "f1@1": 0.07895564642969487, + "f1@5": 0.061340044039005975, + "f1@10": 0.04529726329034287, + "map@1": 0.07895564642969487, + "map@5": 0.11498374750969906, + "map@10": 0.12359781702342257, + "mrr@1": 0.07895564642969487, + "mrr@5": 0.11498374750969906, + "mrr@10": 0.12359781702342257, + "num_pred": 3179, + "num_data": 3179 + }, + "QVHighlight": { + "hit@1": 0.4358264081255771, + "hit@5": 0.8116343490304709, + "hit@10": 1.0, + "ndcg_linear@1": 0.4358264081255771, + "ndcg_linear@5": 0.6322706491755596, + "ndcg_linear@10": 0.6929622320450238, + "ndcg_exponential@1": 0.4358264081255771, + "ndcg_exponential@5": 0.6322706491755596, + "ndcg_exponential@10": 0.6929622320450238, + "precision@1": 0.4358264081255771, + "precision@5": 0.1623268698060942, + "precision@10": 0.09999999999999998, + "recall@1": 0.4358264081255771, + "recall@5": 0.8116343490304709, + "recall@10": 1.0, + "f1@1": 0.4358264081255771, + "f1@5": 0.270544783010157, + "f1@10": 0.18181818181818185, + "map@1": 0.4358264081255771, + "map@5": 0.5728685749461373, + "map@10": 0.5977850327573319, + "mrr@1": 0.4358264081255771, + "mrr@5": 0.5728685749461373, + "mrr@10": 0.5977850327573319, + "num_pred": 1083, + "num_data": 1083 + }, + "Charades-STA": { + "hit@1": 0.1485557083906465, + "hit@5": 0.5281980742778541, + "hit@10": 1.0, + "ndcg_linear@1": 0.1485557083906465, + "ndcg_linear@5": 0.336037308583379, + "ndcg_linear@10": 0.4847984333087624, + "ndcg_exponential@1": 0.1485557083906465, + "ndcg_exponential@5": 0.336037308583379, + "ndcg_exponential@10": 0.4847984333087624, + "precision@1": 0.1485557083906465, + "precision@5": 0.10563961485557083, + "precision@10": 0.09999999999999998, + "recall@1": 0.1485557083906465, + "recall@5": 0.5281980742778541, + "recall@10": 1.0, + "f1@1": 0.1485557083906465, + "f1@5": 0.17606602475928476, + "f1@10": 0.18181818181818185, + "map@1": 0.1485557083906465, + "map@5": 0.27345254470426406, + "map@10": 0.33267013820658936, + "mrr@1": 0.1485557083906465, + "mrr@5": 0.27345254470426406, + "mrr@10": 0.33267013820658936, + "num_pred": 727, + "num_data": 727 + }, + "MomentSeeker": { + "hit@1": 0.3477777777777778, + "hit@5": 0.8511111111111112, + "hit@10": 1.0, + "ndcg_linear@1": 0.3477777777777778, + "ndcg_linear@5": 0.6067492115086224, + "ndcg_linear@10": 0.6630917380371577, + "ndcg_exponential@1": 0.3477777777777778, + "ndcg_exponential@5": 0.6067492115086224, + "ndcg_exponential@10": 0.6630917380371577, + "precision@1": 0.3477777777777778, + "precision@5": 0.18533333333333335, + "precision@10": 0.11305555555555558, + "recall@1": 0.32458333333333333, + "recall@5": 0.8322685185185186, + "recall@10": 0.9990740740740741, + "f1@1": 0.33135273368606705, + "f1@5": 0.29835491668825004, + "f1@10": 0.20046581504914834, + "map@1": 0.3477777777777778, + "map@5": 0.5258817901234568, + "map@10": 0.5519251543209877, + "mrr@1": 0.3477777777777778, + "mrr@5": 0.5379629629629631, + "mrr@10": 0.5581768077601411, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.774, + "hit@5": 0.878, + "hit@10": 0.91, + "ndcg_linear@1": 0.774, + "ndcg_linear@5": 0.8276032117095806, + "ndcg_linear@10": 0.8379559054452919, + "ndcg_exponential@1": 0.774, + "ndcg_exponential@5": 0.8276032117095806, + "ndcg_exponential@10": 0.8379559054452919, + "precision@1": 0.774, + "precision@5": 0.1756, + "precision@10": 0.091, + "recall@1": 0.774, + "recall@5": 0.878, + "recall@10": 0.91, + "f1@1": 0.774, + "f1@5": 0.2926666666666667, + "f1@10": 0.16545454545454544, + "map@1": 0.774, + "map@5": 0.811, + "map@10": 0.8152769841269842, + "mrr@1": 0.774, + "mrr@5": 0.811, + "mrr@10": 0.8152769841269842, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.4434589800443459, + "hit@5": 0.6075388026607539, + "hit@10": 0.6651884700665188, + "ndcg_linear@1": 0.4434589800443459, + "ndcg_linear@5": 0.5311135724096901, + "ndcg_linear@10": 0.549126558201007, + "ndcg_exponential@1": 0.4434589800443459, + "ndcg_exponential@5": 0.5311135724096901, + "ndcg_exponential@10": 0.549126558201007, + "precision@1": 0.4434589800443459, + "precision@5": 0.12416851441241683, + "precision@10": 0.06829268292682926, + "recall@1": 0.437250554323725, + "recall@5": 0.6023281596452328, + "recall@10": 0.6581116038433111, + "f1@1": 0.43902439024390244, + "f1@5": 0.20455818691112812, + "f1@10": 0.12296562762194693, + "map@1": 0.4434589800443459, + "map@5": 0.505469327420547, + "map@10": 0.512853601344455, + "mrr@1": 0.4434589800443459, + "mrr@5": 0.5099408721359941, + "mrr@10": 0.5174251223031711, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.8481781376518218, + "hit@5": 0.9453441295546559, + "hit@10": 0.9615384615384616, + "ndcg_linear@1": 0.8481781376518218, + "ndcg_linear@5": 0.9016308878511893, + "ndcg_linear@10": 0.9073181214772046, + "ndcg_exponential@1": 0.8481781376518218, + "ndcg_exponential@5": 0.9016308878511893, + "ndcg_exponential@10": 0.9073181214772046, + "precision@1": 0.8481781376518218, + "precision@5": 0.1890688259109312, + "precision@10": 0.09635627530364373, + "recall@1": 0.8481781376518218, + "recall@5": 0.9427125506072874, + "recall@10": 0.9599190283400809, + "f1@1": 0.8481781376518218, + "f1@5": 0.3147484094852516, + "f1@10": 0.17503373819163293, + "map@1": 0.8481781376518218, + "map@5": 0.8873481781376519, + "map@10": 0.8898359681254419, + "mrr@1": 0.8481781376518218, + "mrr@5": 0.8883603238866398, + "mrr@10": 0.8905589293747189, + "num_pred": 494, + "num_data": 494 + }, + "ViDoRe_tabfquad": { + "hit@1": 0.8892857142857142, + "hit@5": 0.9678571428571429, + "hit@10": 0.975, + "ndcg_linear@1": 0.8892857142857142, + "ndcg_linear@5": 0.9332834391651387, + "ndcg_linear@10": 0.9356005756298494, + "ndcg_exponential@1": 0.8892857142857142, + "ndcg_exponential@5": 0.9332834391651387, + "ndcg_exponential@10": 0.9356005756298494, + "precision@1": 0.8892857142857142, + "precision@5": 0.19357142857142853, + "precision@10": 0.09749999999999999, + "recall@1": 0.8892857142857142, + "recall@5": 0.9678571428571429, + "recall@10": 0.975, + "f1@1": 0.8892857142857142, + "f1@5": 0.3226190476190477, + "f1@10": 0.1772727272727273, + "map@1": 0.8892857142857142, + "map@5": 0.921547619047619, + "map@10": 0.9225042517006803, + "mrr@1": 0.8892857142857142, + "mrr@5": 0.921547619047619, + "mrr@10": 0.9225042517006803, + "num_pred": 280, + "num_data": 280 }, - "MSVD": { - "hit@1": 0.4761194029850746, - "hit@5": 0.753731343283582, - "hit@10": 0.8223880597014925, - "ndcg_linear@1": 0.4761194029850746, - "ndcg_linear@5": 0.6277485828345324, - "ndcg_linear@10": 0.6502611190764599, - "ndcg_exponential@1": 0.4761194029850746, - "ndcg_exponential@5": 0.6277485828345324, - "ndcg_exponential@10": 0.6502611190764599, - "precision@1": 0.4761194029850746, - "precision@5": 0.15074626865671642, - "precision@10": 0.08223880597014925, - "recall@1": 0.4761194029850746, - "recall@5": 0.753731343283582, - "recall@10": 0.8223880597014925, - "f1@1": 0.4761194029850746, - "f1@5": 0.25124378109452744, - "f1@10": 0.14952510176390776, - "map@1": 0.4761194029850746, - "map@5": 0.5854726368159203, - "map@10": 0.5949484719260838, - "mrr@1": 0.4761194029850746, - "mrr@5": 0.5854726368159203, - "mrr@10": 0.5949484719260838, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tatdqa": { + "hit@1": 0.5577156743620899, + "hit@5": 0.8250303766707169, + "hit@10": 0.8894289185905225, + "ndcg_linear@1": 0.5577156743620899, + "ndcg_linear@5": 0.6993890766047229, + "ndcg_linear@10": 0.7206656381690775, + "ndcg_exponential@1": 0.5577156743620899, + "ndcg_exponential@5": 0.6993890766047229, + "ndcg_exponential@10": 0.7206656381690775, + "precision@1": 0.5577156743620899, + "precision@5": 0.16537059538274604, + "precision@10": 0.08924665856622113, + "recall@1": 0.5562575941676793, + "recall@5": 0.8235722964763061, + "recall@10": 0.8885783718104495, + "f1@1": 0.556611068154203, + "f1@5": 0.2752357808250883, + "f1@10": 0.1621156890901727, + "map@1": 0.5577156743620899, + "map@5": 0.6577612393681652, + "map@10": 0.6667486354606647, + "mrr@1": 0.5577156743620899, + "mrr@5": 0.6586776022681248, + "mrr@10": 0.6675377056452391, + "num_pred": 1646, + "num_data": 1646 }, - "VATEX": { - "hit@1": 0.22956677087985708, - "hit@5": 0.44796784278695845, - "hit@10": 0.5482358195623046, - "ndcg_linear@1": 0.22956677087985708, - "ndcg_linear@5": 0.3442450550592858, - "ndcg_linear@10": 0.37658573093046976, - "ndcg_exponential@1": 0.22956677087985708, - "ndcg_exponential@5": 0.3442450550592858, - "ndcg_exponential@10": 0.37658573093046976, - "precision@1": 0.22956677087985708, - "precision@5": 0.0895935685573917, - "precision@10": 0.05482358195623046, - "recall@1": 0.22956677087985708, - "recall@5": 0.44796784278695845, - "recall@10": 0.5482358195623046, - "f1@1": 0.22956677087985708, - "f1@5": 0.1493226142623195, - "f1@10": 0.09967923992041902, - "map@1": 0.22956677087985708, - "map@5": 0.3098518683936281, - "map@10": 0.3231518109700334, - "mrr@1": 0.22956677087985708, - "mrr@5": 0.3098518683936281, - "mrr@10": 0.3231518109700334, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_shiftproject": { + "hit@1": 0.8, + "hit@5": 0.97, + "hit@10": 0.99, + "ndcg_linear@1": 0.8, + "ndcg_linear@5": 0.894703974563633, + "ndcg_linear@10": 0.9010476078536062, + "ndcg_exponential@1": 0.8, + "ndcg_exponential@5": 0.894703974563633, + "ndcg_exponential@10": 0.9010476078536062, + "precision@1": 0.8, + "precision@5": 0.19399999999999998, + "precision@10": 0.09899999999999999, + "recall@1": 0.8, + "recall@5": 0.97, + "recall@10": 0.99, + "f1@1": 0.8, + "f1@5": 0.3233333333333334, + "f1@10": 0.18000000000000008, + "map@1": 0.8, + "map@5": 0.8691666666666665, + "map@10": 0.8717063492063492, + "mrr@1": 0.8, + "mrr@5": 0.8691666666666665, + "mrr@10": 0.8717063492063492, + "num_pred": 100, + "num_data": 100 }, - "YouCook2": { - "hit@1": 0.07895564642969487, - "hit@5": 0.18402013211701793, - "hit@10": 0.2491349480968858, - "ndcg_linear@1": 0.07895564642969487, - "ndcg_linear@5": 0.13201194607861527, - "ndcg_linear@10": 0.1529921311761448, - "ndcg_exponential@1": 0.07895564642969487, - "ndcg_exponential@5": 0.13201194607861527, - "ndcg_exponential@10": 0.1529921311761448, - "precision@1": 0.07895564642969487, - "precision@5": 0.03680402642340359, - "precision@10": 0.02491349480968858, - "recall@1": 0.07895564642969487, - "recall@5": 0.18402013211701793, - "recall@10": 0.2491349480968858, - "f1@1": 0.07895564642969487, - "f1@5": 0.061340044039005975, - "f1@10": 0.04529726329034287, - "map@1": 0.07895564642969487, - "map@5": 0.11498374750969906, - "map@10": 0.12359781702342257, - "mrr@1": 0.07895564642969487, - "mrr@5": 0.11498374750969906, - "mrr@10": 0.12359781702342257, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.96, + "hit@5": 0.99, + "hit@10": 1.0, + "ndcg_linear@1": 0.96, + "ndcg_linear@5": 0.97517782560806, + "ndcg_linear@10": 0.9787398974791403, + "ndcg_exponential@1": 0.96, + "ndcg_exponential@5": 0.97517782560806, + "ndcg_exponential@10": 0.9787398974791403, + "precision@1": 0.96, + "precision@5": 0.19799999999999998, + "precision@10": 0.09999999999999998, + "recall@1": 0.96, + "recall@5": 0.99, + "recall@10": 1.0, + "f1@1": 0.96, + "f1@5": 0.3300000000000001, + "f1@10": 0.1818181818181819, + "map@1": 0.96, + "map@5": 0.9703333333333333, + "map@10": 0.972, + "mrr@1": 0.96, + "mrr@5": 0.9703333333333333, + "mrr@10": 0.972, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.88, + "hit@5": 0.95, + "hit@10": 0.96, + "ndcg_linear@1": 0.88, + "ndcg_linear@5": 0.9191057182152036, + "ndcg_linear@10": 0.9221160181718434, + "ndcg_exponential@1": 0.88, + "ndcg_exponential@5": 0.9191057182152036, + "ndcg_exponential@10": 0.9221160181718434, + "precision@1": 0.88, + "precision@5": 0.18999999999999997, + "precision@10": 0.09599999999999997, + "recall@1": 0.88, + "recall@5": 0.95, + "recall@10": 0.96, + "f1@1": 0.88, + "f1@5": 0.3166666666666667, + "f1@10": 0.17454545454545464, + "map@1": 0.88, + "map@5": 0.9086666666666667, + "map@10": 0.9097777777777778, + "mrr@1": 0.88, + "mrr@5": 0.9086666666666667, + "mrr@10": 0.9097777777777778, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.4358264081255771, - "hit@5": 0.8116343490304709, - "hit@10": 1.0, - "ndcg_linear@1": 0.4358264081255771, - "ndcg_linear@5": 0.6322706491755596, - "ndcg_linear@10": 0.6929622320450238, - "ndcg_exponential@1": 0.4358264081255771, - "ndcg_exponential@5": 0.6322706491755596, - "ndcg_exponential@10": 0.6929622320450238, - "precision@1": 0.4358264081255771, - "precision@5": 0.1623268698060942, - "precision@10": 0.09999999999999998, - "recall@1": 0.4358264081255771, - "recall@5": 0.8116343490304709, - "recall@10": 1.0, - "f1@1": 0.4358264081255771, - "f1@5": 0.270544783010157, - "f1@10": 0.18181818181818185, - "map@1": 0.4358264081255771, - "map@5": 0.5728685749461373, - "map@10": 0.5977850327573319, - "mrr@1": 0.4358264081255771, - "mrr@5": 0.5728685749461373, - "mrr@10": 0.5977850327573319, - "num_pred": 1083, - "num_data": 1083 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.91, + "hit@5": 0.97, + "hit@10": 0.98, + "ndcg_linear@1": 0.91, + "ndcg_linear@5": 0.9458532532593068, + "ndcg_linear@10": 0.9487439015224857, + "ndcg_exponential@1": 0.91, + "ndcg_exponential@5": 0.9458532532593068, + "ndcg_exponential@10": 0.9487439015224857, + "precision@1": 0.91, + "precision@5": 0.19399999999999995, + "precision@10": 0.09799999999999998, + "recall@1": 0.91, + "recall@5": 0.97, + "recall@10": 0.98, + "f1@1": 0.91, + "f1@5": 0.3233333333333334, + "f1@10": 0.17818181818181827, + "map@1": 0.91, + "map@5": 0.9375, + "map@10": 0.9384999999999999, + "mrr@1": 0.91, + "mrr@5": 0.9375, + "mrr@10": 0.9384999999999999, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.1485557083906465, - "hit@5": 0.5281980742778541, + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.97, + "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.1485557083906465, - "ndcg_linear@5": 0.336037308583379, - "ndcg_linear@10": 0.4847984333087624, - "ndcg_exponential@1": 0.1485557083906465, - "ndcg_exponential@5": 0.336037308583379, - "ndcg_exponential@10": 0.4847984333087624, - "precision@1": 0.1485557083906465, - "precision@5": 0.10563961485557083, + "ndcg_linear@1": 0.97, + "ndcg_linear@5": 0.9869253606521631, + "ndcg_linear@10": 0.9869253606521631, + "ndcg_exponential@1": 0.97, + "ndcg_exponential@5": 0.9869253606521631, + "ndcg_exponential@10": 0.9869253606521631, + "precision@1": 0.97, + "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.1485557083906465, - "recall@5": 0.5281980742778541, + "recall@1": 0.97, + "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.1485557083906465, - "f1@5": 0.17606602475928476, - "f1@10": 0.18181818181818185, - "map@1": 0.1485557083906465, - "map@5": 0.27345254470426406, - "map@10": 0.33267013820658936, - "mrr@1": 0.1485557083906465, - "mrr@5": 0.27345254470426406, - "mrr@10": 0.33267013820658936, - "num_pred": 727, - "num_data": 727 + "f1@1": 0.97, + "f1@5": 0.3333333333333335, + "f1@10": 0.1818181818181819, + "map@1": 0.97, + "map@5": 0.9825, + "map@10": 0.9825, + "mrr@1": 0.97, + "mrr@5": 0.9825, + "mrr@10": 0.9825, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.387, - "hit@5": 0.839, - "hit@10": 1.0, - "ndcg_linear@1": 0.387, - "ndcg_linear@5": 0.6218528020441433, - "ndcg_linear@10": 0.6764150062417216, - "ndcg_exponential@1": 0.387, - "ndcg_exponential@5": 0.6218528020441433, - "ndcg_exponential@10": 0.6764150062417216, - "precision@1": 0.387, - "precision@5": 0.17640000000000003, - "precision@10": 0.10660000000000003, - "recall@1": 0.3735833333333334, - "recall@5": 0.834, - "recall@10": 1.0, - "f1@1": 0.37714920634920635, - "f1@5": 0.28777189477189485, - "f1@10": 0.19060245310245308, - "map@1": 0.387, - "map@5": 0.5495050000000001, - "map@10": 0.5732019841269842, - "mrr@1": 0.387, - "mrr@5": 0.5534333333333333, - "mrr@10": 0.5752948412698413, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.6153846153846154, + "hit@5": 0.7884615384615384, + "hit@10": 0.8653846153846154, + "ndcg_linear@1": 0.5897435897435896, + "ndcg_linear@5": 0.6094691508995544, + "ndcg_linear@10": 0.635637750038373, + "ndcg_exponential@1": 0.5824175824175823, + "ndcg_exponential@5": 0.6103242475099454, + "ndcg_exponential@10": 0.6356245865615132, + "precision@1": 0.6153846153846154, + "precision@5": 0.23461538461538461, + "precision@10": 0.1423076923076923, + "recall@1": 0.4133241758241758, + "recall@5": 0.6478021978021977, + "recall@10": 0.7252372627372627, + "f1@1": 0.4552021696252465, + "f1@5": 0.3082283729342553, + "f1@10": 0.2158892483835923, + "map@1": 0.6153846153846154, + "map@5": 0.5464262820512821, + "map@10": 0.5583285002035001, + "mrr@1": 0.6153846153846154, + "mrr@5": 0.6769230769230768, + "mrr@10": 0.6880036630036629, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.352, - "hit@5": 0.558, - "hit@10": 0.634, - "ndcg_linear@1": 0.352, - "ndcg_linear@5": 0.4624292100503551, - "ndcg_linear@10": 0.4871702819761314, - "ndcg_exponential@1": 0.352, - "ndcg_exponential@5": 0.4624292100503551, - "ndcg_exponential@10": 0.4871702819761314, - "precision@1": 0.352, - "precision@5": 0.11159999999999999, - "precision@10": 0.06339999999999998, - "recall@1": 0.352, - "recall@5": 0.558, - "recall@10": 0.634, - "f1@1": 0.352, - "f1@5": 0.18600000000000005, - "f1@10": 0.11527272727272728, - "map@1": 0.352, - "map@5": 0.4306, - "map@10": 0.440915873015873, - "mrr@1": 0.352, - "mrr@5": 0.4306, - "mrr@10": 0.440915873015873, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.490625, + "hit@5": 0.796875, + "hit@10": 0.86875, + "ndcg_linear@1": 0.490625, + "ndcg_linear@5": 0.5397544908352085, + "ndcg_linear@10": 0.5693203555054913, + "ndcg_exponential@1": 0.490625, + "ndcg_exponential@5": 0.5397544908352085, + "ndcg_exponential@10": 0.5693203555054912, + "precision@1": 0.490625, + "precision@5": 0.251875, + "precision@10": 0.16140625000000003, + "recall@1": 0.2978885162019353, + "recall@5": 0.5770897790934556, + "recall@10": 0.6719354787369494, + "f1@1": 0.3395271915584416, + "f1@5": 0.3084593392718393, + "f1@10": 0.23304521375615125, + "map@1": 0.490625, + "map@5": 0.46664019097222215, + "map@10": 0.479486573188303, + "mrr@1": 0.490625, + "mrr@5": 0.6060416666666667, + "mrr@10": 0.6155078125, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.299, - "hit@5": 0.528, - "hit@10": 0.622, - "ndcg_linear@1": 0.299, - "ndcg_linear@5": 0.41899491853941084, - "ndcg_linear@10": 0.44961784439388147, - "ndcg_exponential@1": 0.299, - "ndcg_exponential@5": 0.41899491853941084, - "ndcg_exponential@10": 0.44961784439388147, - "precision@1": 0.299, - "precision@5": 0.1056, - "precision@10": 0.0622, - "recall@1": 0.299, - "recall@5": 0.528, - "recall@10": 0.622, - "f1@1": 0.299, - "f1@5": 0.17600000000000005, - "f1@10": 0.11309090909090909, - "map@1": 0.299, - "map@5": 0.38288333333333335, - "map@10": 0.3956539682539683, - "mrr@1": 0.299, - "mrr@5": 0.38288333333333335, - "mrr@10": 0.3956539682539683, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.5689655172413793, + "hit@5": 0.8491379310344828, + "hit@10": 0.8879310344827587, + "ndcg_linear@1": 0.5689655172413793, + "ndcg_linear@5": 0.5019343305838408, + "ndcg_linear@10": 0.47979676122532555, + "ndcg_exponential@1": 0.5689655172413793, + "ndcg_exponential@5": 0.5019343305838408, + "ndcg_exponential@10": 0.47979676122532555, + "precision@1": 0.5689655172413793, + "precision@5": 0.428448275862069, + "precision@10": 0.34698275862068967, + "recall@1": 0.08034274543141959, + "recall@5": 0.25237595480423564, + "recall@10": 0.3645843669588603, + "f1@1": 0.11905920072661083, + "f1@5": 0.25183497028873714, + "f1@10": 0.2856798311584853, + "map@1": 0.5689655172413793, + "map@5": 0.40593749999999995, + "map@10": 0.35002105419913815, + "mrr@1": 0.5689655172413793, + "mrr@5": 0.6777298850574713, + "mrr@10": 0.682835591133005, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.4473684210526316, + "hit@5": 0.793859649122807, + "hit@10": 0.8771929824561403, + "ndcg_linear@1": 0.4473684210526316, + "ndcg_linear@5": 0.5073413437187128, + "ndcg_linear@10": 0.5436696652694063, + "ndcg_exponential@1": 0.4473684210526316, + "ndcg_exponential@5": 0.5073413437187128, + "ndcg_exponential@10": 0.5436696652694063, + "precision@1": 0.4473684210526316, + "precision@5": 0.2429824561403509, + "precision@10": 0.1635964912280702, + "recall@1": 0.2503968253968254, + "recall@5": 0.5527621136173767, + "recall@10": 0.6586466165413534, + "f1@1": 0.29117386075643786, + "f1@5": 0.2958373843475797, + "f1@10": 0.22844515727846343, + "map@1": 0.4473684210526316, + "map@5": 0.4241118421052631, + "map@10": 0.4392249141371949, + "mrr@1": 0.4473684210526316, + "mrr@5": 0.5828216374269005, + "mrr@10": 0.5950205374547479, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.434, - "hit@5": 0.672, - "hit@10": 0.768, - "ndcg_linear@1": 0.434, - "ndcg_linear@5": 0.5564379559041219, - "ndcg_linear@10": 0.587723216959859, - "ndcg_exponential@1": 0.434, - "ndcg_exponential@5": 0.5564379559041219, - "ndcg_exponential@10": 0.587723216959859, - "precision@1": 0.434, - "precision@5": 0.13439999999999996, - "precision@10": 0.0768, - "recall@1": 0.434, - "recall@5": 0.672, - "recall@10": 0.768, - "f1@1": 0.434, - "f1@5": 0.22400000000000006, - "f1@10": 0.13963636363636361, - "map@1": 0.434, - "map@5": 0.5183166666666666, - "map@10": 0.531368253968254, - "mrr@1": 0.434, - "mrr@5": 0.5183166666666666, - "mrr@10": 0.531368253968254, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.7610294117647058, + "hit@5": 0.8713235294117647, + "hit@10": 0.9068627450980392, + "ndcg_linear@1": 0.7610294117647058, + "ndcg_linear@5": 0.8199677874725917, + "ndcg_linear@10": 0.8315054893421978, + "ndcg_exponential@1": 0.7610294117647058, + "ndcg_exponential@5": 0.8199677874725917, + "ndcg_exponential@10": 0.8315054893421978, + "precision@1": 0.7610294117647058, + "precision@5": 0.17426470588235293, + "precision@10": 0.09068627450980392, + "recall@1": 0.7610294117647058, + "recall@5": 0.8713235294117647, + "recall@10": 0.9068627450980392, + "f1@1": 0.7610294117647058, + "f1@5": 0.2904411764705883, + "f1@10": 0.1648841354723708, + "map@1": 0.7610294117647058, + "map@5": 0.8028799019607843, + "map@10": 0.8076700124494243, + "mrr@1": 0.7610294117647058, + "mrr@5": 0.8028799019607843, + "mrr@10": 0.8076700124494243, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.524, - "hit@5": 0.745, - "hit@10": 0.821, - "ndcg_linear@1": 0.524, - "ndcg_linear@5": 0.6452214501789469, - "ndcg_linear@10": 0.6701386847278763, - "ndcg_exponential@1": 0.524, - "ndcg_exponential@5": 0.6452214501789469, - "ndcg_exponential@10": 0.6701386847278763, - "precision@1": 0.524, - "precision@5": 0.149, - "precision@10": 0.0821, - "recall@1": 0.524, - "recall@5": 0.745, - "recall@10": 0.821, - "f1@1": 0.524, - "f1@5": 0.2483333333333334, - "f1@10": 0.14927272727272728, - "map@1": 0.524, - "map@5": 0.6116833333333334, - "map@10": 0.6221575396825396, - "mrr@1": 0.524, - "mrr@5": 0.6116833333333334, - "mrr@10": 0.6221575396825396, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.6825396825396826, + "hit@5": 0.9047619047619048, + "hit@10": 0.9206349206349206, + "ndcg_linear@1": 0.6825396825396826, + "ndcg_linear@5": 0.7988138511124628, + "ndcg_linear@10": 0.8034021816889372, + "ndcg_exponential@1": 0.6825396825396826, + "ndcg_exponential@5": 0.7988138511124628, + "ndcg_exponential@10": 0.8034021816889372, + "precision@1": 0.6825396825396826, + "precision@5": 0.18095238095238086, + "precision@10": 0.09206349206349201, + "recall@1": 0.6825396825396826, + "recall@5": 0.9047619047619048, + "recall@10": 0.9206349206349206, + "f1@1": 0.6825396825396826, + "f1@5": 0.3015873015873015, + "f1@10": 0.16738816738816742, + "map@1": 0.6825396825396826, + "map@5": 0.7640211640211642, + "map@10": 0.7656084656084656, + "mrr@1": 0.6825396825396826, + "mrr@5": 0.7640211640211642, + "mrr@10": 0.7656084656084656, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.13625866050808313, - "hit@5": 0.5935334872979214, - "hit@10": 1.0, - "ndcg_linear@1": 0.13625866050808313, - "ndcg_linear@5": 0.3556714764778745, - "ndcg_linear@10": 0.4859126067666473, - "ndcg_exponential@1": 0.13625866050808313, - "ndcg_exponential@5": 0.3556714764778745, - "ndcg_exponential@10": 0.4859126067666473, - "precision@1": 0.13625866050808313, - "precision@5": 0.11870669745958429, - "precision@10": 0.1, - "recall@1": 0.13625866050808313, - "recall@5": 0.5935334872979214, - "recall@10": 1.0, - "f1@1": 0.13625866050808313, - "f1@5": 0.19784449576597388, - "f1@10": 0.1818181818181818, - "map@1": 0.13625866050808313, - "map@5": 0.27886836027713624, - "map@10": 0.3318990798782947, - "mrr@1": 0.13625866050808313, - "mrr@5": 0.27886836027713624, - "mrr@10": 0.3318990798782947, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.7648054145516074, + "hit@5": 0.9035532994923858, + "hit@10": 0.937394247038917, + "ndcg_linear@1": 0.7648054145516074, + "ndcg_linear@5": 0.8442477575170283, + "ndcg_linear@10": 0.8553129247371343, + "ndcg_exponential@1": 0.7648054145516074, + "ndcg_exponential@5": 0.8442477575170283, + "ndcg_exponential@10": 0.8553129247371343, + "precision@1": 0.7648054145516074, + "precision@5": 0.18071065989847712, + "precision@10": 0.0937394247038917, + "recall@1": 0.7648054145516074, + "recall@5": 0.9035532994923858, + "recall@10": 0.937394247038917, + "f1@1": 0.7648054145516074, + "f1@5": 0.3011844331641287, + "f1@10": 0.1704353176434395, + "map@1": 0.7648054145516074, + "map@5": 0.8239988719684151, + "map@10": 0.8286332017296485, + "mrr@1": 0.7648054145516074, + "mrr@5": 0.8239988719684151, + "mrr@10": 0.8286332017296485, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.3745, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3745, - "ndcg_linear@5": 0.7155447146201682, - "ndcg_linear@10": 0.7155447146201682, - "ndcg_exponential@1": 0.3745, - "ndcg_exponential@5": 0.7155447146201682, - "ndcg_exponential@10": 0.7155447146201682, - "precision@1": 0.3745, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.3745, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3745, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.3745, - "map@5": 0.6196291666666666, - "map@10": 0.6196291666666666, - "mrr@1": 0.3745, - "mrr@5": 0.6196291666666666, - "mrr@10": 0.6196291666666666, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.9064748201438849, + "hit@5": 0.9802158273381295, + "hit@10": 0.9856115107913669, + "ndcg_linear@1": 0.9064748201438849, + "ndcg_linear@5": 0.9337625442422413, + "ndcg_linear@10": 0.9378099940259211, + "ndcg_exponential@1": 0.9064748201438849, + "ndcg_exponential@5": 0.9337625442422413, + "ndcg_exponential@10": 0.9378099940259211, + "precision@1": 0.9064748201438849, + "precision@5": 0.24244604316546764, + "precision@10": 0.12302158273381293, + "recall@1": 0.783273381294964, + "recall@5": 0.966726618705036, + "recall@10": 0.9775179856115108, + "f1@1": 0.8243405275779376, + "f1@5": 0.380695443645084, + "f1@10": 0.21601809461521695, + "map@1": 0.9064748201438849, + "map@5": 0.9146432853717026, + "map@10": 0.9169378782688133, + "mrr@1": 0.9064748201438849, + "mrr@5": 0.9376199040767385, + "mrr@10": 0.938401421719767, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.8426183844011143, + "hit@5": 0.9651810584958217, + "hit@10": 0.9874651810584958, + "ndcg_linear@1": 0.8426183844011143, + "ndcg_linear@5": 0.9135602456956691, + "ndcg_linear@10": 0.9210168523969697, + "ndcg_exponential@1": 0.8426183844011143, + "ndcg_exponential@5": 0.9135602456956691, + "ndcg_exponential@10": 0.9210168523969697, + "precision@1": 0.8426183844011143, + "precision@5": 0.1930362116991643, + "precision@10": 0.09874651810584957, + "recall@1": 0.8426183844011143, + "recall@5": 0.9651810584958217, + "recall@10": 0.9874651810584958, + "f1@1": 0.8426183844011143, + "f1@5": 0.3217270194986073, + "f1@10": 0.17953912382881745, + "map@1": 0.8426183844011143, + "map@5": 0.8958681522748375, + "map@10": 0.8990941548392801, + "mrr@1": 0.8426183844011143, + "mrr@5": 0.8958681522748375, + "mrr@10": 0.8990941548392801, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.3425925925925926, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3425925925925926, - "ndcg_linear@5": 0.6913650713075382, - "ndcg_linear@10": 0.6913650713075382, - "ndcg_exponential@1": 0.3425925925925926, - "ndcg_exponential@5": 0.6913650713075382, - "ndcg_exponential@10": 0.6913650713075382, - "precision@1": 0.3425925925925926, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.3425925925925926, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3425925925925926, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.3425925925925926, - "map@5": 0.5881790123456789, - "map@10": 0.5881790123456789, - "mrr@1": 0.3425925925925926, - "mrr@5": 0.5881790123456789, - "mrr@10": 0.5881790123456789, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.5052143684820394, + "hit@5": 0.764774044032445, + "hit@10": 0.8551564310544612, + "ndcg_linear@1": 0.5052143684820394, + "ndcg_linear@5": 0.640903570079321, + "ndcg_linear@10": 0.6707054604251023, + "ndcg_exponential@1": 0.5052143684820394, + "ndcg_exponential@5": 0.640903570079321, + "ndcg_exponential@10": 0.6707054604251023, + "precision@1": 0.5052143684820394, + "precision@5": 0.15295480880648896, + "precision@10": 0.0855156431054461, + "recall@1": 0.5052143684820394, + "recall@5": 0.764774044032445, + "recall@10": 0.8551564310544612, + "f1@1": 0.5052143684820394, + "f1@5": 0.25492468134414836, + "f1@10": 0.15548298746444753, + "map@1": 0.5052143684820394, + "map@5": 0.5998648126689843, + "map@10": 0.612502069193842, + "mrr@1": 0.5052143684820394, + "mrr@5": 0.5998648126689843, + "mrr@10": 0.612502069193842, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.3949089210649229, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.3949089210649229, - "ndcg_linear@5": 0.7033624022532277, - "ndcg_linear@10": 0.7033624022532277, - "ndcg_exponential@1": 0.3949089210649229, - "ndcg_exponential@5": 0.7033624022532277, - "ndcg_exponential@10": 0.7033624022532277, - "precision@1": 0.3949089210649229, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.3949089210649229, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.3949089210649229, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.3949089210649229, - "map@5": 0.605927915304375, - "map@10": 0.605927915304375, - "mrr@1": 0.3949089210649229, - "mrr@5": 0.6057527635061498, - "mrr@10": 0.6057527635061498, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.01838879159369527, + "hit@5": 0.4089316987740806, + "hit@10": 0.6401050788091068, + "ndcg_linear@1": 0.01838879159369527, + "ndcg_linear@5": 0.21620008992198805, + "ndcg_linear@10": 0.2907931583080436, + "ndcg_exponential@1": 0.01838879159369527, + "ndcg_exponential@5": 0.21620008992198805, + "ndcg_exponential@10": 0.2907931583080436, + "precision@1": 0.01838879159369527, + "precision@5": 0.08178633975481611, + "precision@10": 0.06401050788091067, + "recall@1": 0.01838879159369527, + "recall@5": 0.4089316987740806, + "recall@10": 0.6401050788091068, + "f1@1": 0.01838879159369527, + "f1@5": 0.13631056625802687, + "f1@10": 0.1163827416016558, + "map@1": 0.01838879159369527, + "map@5": 0.15302101576182137, + "map@10": 0.1837190392794596, + "mrr@1": 0.01838879159369527, + "mrr@5": 0.15302101576182137, + "mrr@10": 0.1837190392794596, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.408, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.408, - "ndcg_linear@5": 0.7172123608985213, - "ndcg_linear@10": 0.7172123608985213, - "ndcg_exponential@1": 0.408, - "ndcg_exponential@5": 0.7172123608985213, - "ndcg_exponential@10": 0.7172123608985213, - "precision@1": 0.408, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.408, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.408, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.408, - "map@5": 0.6235, - "map@10": 0.6235, - "mrr@1": 0.408, - "mrr@5": 0.6235, - "mrr@10": 0.6235, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.9964973730297724, + "hit@5": 0.9982486865148862, + "hit@10": 0.9982486865148862, + "ndcg_linear@1": 0.678342089900759, + "ndcg_linear@5": 0.8362374075645319, + "ndcg_linear@10": 0.8165935940149865, + "ndcg_exponential@1": 0.4510883162371779, + "ndcg_exponential@5": 0.7145068150665362, + "ndcg_exponential@10": 0.7410897811756266, + "precision@1": 0.9964973730297724, + "precision@5": 0.9262697022767075, + "precision@10": 0.8286339754816112, + "recall@1": 0.05876920449269009, + "recall@5": 0.2726143296172191, + "recall@10": 0.4766810037927378, + "f1@1": 0.11024247532054007, + "f1@5": 0.4126599828359686, + "f1@10": 0.5917511632990256, + "map@1": 0.9964973730297724, + "map@5": 0.9193257443082313, + "map@10": 0.8188673981787047, + "mrr@1": 0.9964973730297724, + "mrr@5": 0.997008172796264, + "mrr@10": 0.997008172796264, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.58, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.58, - "ndcg_linear@5": 0.8449904965000122, - "ndcg_linear@10": 0.8449904965000122, - "ndcg_exponential@1": 0.58, - "ndcg_exponential@5": 0.8449904965000122, - "ndcg_exponential@10": 0.8449904965000122, - "precision@1": 0.58, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.58, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.58, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.58, - "map@5": 0.79, - "map@10": 0.79, - "mrr@1": 0.58, - "mrr@5": 0.79, - "mrr@10": 0.79, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.07637231503579953, + "hit@5": 0.3031026252983294, + "hit@10": 0.4033412887828162, + "ndcg_linear@1": 0.07637231503579953, + "ndcg_linear@5": 0.15820239639498082, + "ndcg_linear@10": 0.19395841349578616, + "ndcg_exponential@1": 0.07637231503579953, + "ndcg_exponential@5": 0.15820239639498082, + "ndcg_exponential@10": 0.19395841349578616, + "precision@1": 0.07637231503579953, + "precision@5": 0.07159904534606205, + "precision@10": 0.052983293556085914, + "recall@1": 0.048528241845664274, + "recall@5": 0.22104074326628023, + "recall@10": 0.32010313672008184, + "f1@1": 0.05683207007789346, + "f1@5": 0.10227087246601498, + "f1@10": 0.08682064083863344, + "map@1": 0.07637231503579953, + "map@5": 0.11957305754441794, + "map@10": 0.13619920114028108, + "mrr@1": 0.07637231503579953, + "mrr@5": 0.16328560063643593, + "mrr@10": 0.17665170284502024, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.698090692124105, + "hit@5": 0.8007159904534606, + "hit@10": 0.8281622911694511, + "ndcg_linear@1": 0.49562450278440734, + "ndcg_linear@5": 0.5143162653023697, + "ndcg_linear@10": 0.49047862420126026, + "ndcg_exponential@1": 0.35100579611319466, + "ndcg_exponential@5": 0.43113924192770037, + "ndcg_exponential@10": 0.43753218599979815, + "precision@1": 0.698090692124105, + "precision@5": 0.5663484486873509, + "precision@10": 0.4947494033412887, + "recall@1": 0.022961848267359367, + "recall@5": 0.08932041189913144, + "recall@10": 0.15198617349694285, + "f1@1": 0.044005822895889966, + "f1@5": 0.1481301071634879, + "f1@10": 0.217939992851331, + "map@1": 0.698090692124105, + "map@5": 0.5353301511535402, + "map@10": 0.44876554785266, + "mrr@1": 0.698090692124105, + "mrr@5": 0.7382259347653143, + "mrr@10": 0.7417945221047847, + "num_pred": 838, + "num_data": 838 } } }