Upload folder using huggingface_hub

all_tasks.json

@@ -0,0 +1,34 @@
+[
+    {"dataset": "AQUA","subset_id": 0, "task_name": "no external knowledge"},
+    {"dataset": "AQUA","subset_id": 1, "task_name": "need external knowledge 1"},
+    {"dataset": "AQUA","subset_id": 2, "task_name": "need external knowledge 2"},
+    {"dataset": "Mementos","subset_id": 0},
+    {"dataset": "SciCap","subset_id": 0, "task_name": "Yes-subfig"},
+    {"dataset": "SciCap","subset_id": 1, "task_name": "No-subfig"},
+    {"dataset": "Bongard-OpenWorld","subset_id": 0},
+    {"dataset": "HRVQA","subset_id": 0, "task_name": "areas"},
+    {"dataset": "HRVQA","subset_id": 1, "task_name": "color"},
+    {"dataset": "HRVQA","subset_id": 2, "task_name": "location"},
+    {"dataset": "HRVQA","subset_id": 3, "task_name": "numbers"},
+    {"dataset": "HRVQA","subset_id": 4, "task_name": "scene"},
+    {"dataset": "HRVQA","subset_id": 5, "task_name": "shape"},
+    {"dataset": "HRVQA","subset_id": 6, "task_name": "size"},
+    {"dataset": "HRVQA","subset_id": 7, "task_name": "sports"},
+    {"dataset": "HRVQA","subset_id": 8, "task_name": "transportation"},
+    {"dataset": "HRVQA","subset_id": 9, "task_name": "yes no"},
+    {"dataset": "mPLUG","subset_id": 0, "task_name": "areas"},
+    {"dataset": "mPLUG","subset_id": 1, "task_name": "color"},
+    {"dataset": "mPLUG","subset_id": 2, "task_name": "location"},
+    {"dataset": "mPLUG","subset_id": 3, "task_name": "numbers"},
+    {"dataset": "mPLUG","subset_id": 4, "task_name": "scene"},
+    {"dataset": "mPLUG","subset_id": 5, "task_name": "shape"},
+    {"dataset": "mPLUG","subset_id": 6, "task_name": "size"},
+    {"dataset": "mPLUG","subset_id": 7, "task_name": "sports"},
+    {"dataset": "Birds-to-Words","subset_id": 0},
+    {"dataset": "DiDeMoSV","subset_id": 0},
+    {"dataset": "VIST","subset_id": 0},
+    {"dataset": "Describe-Diff","subset_id": 0},
+    {"dataset": "NLVR2","subset_id": 0},
+    {"dataset": "NLVR2","subset_id": 1},
+    {"dataset": "KGQA","subset_id": 0}
+]

continuous/scenario-11.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 150, "type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 150, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 150, "type":"multi-choice"},
+            {"dataset": "TQA","subset_id": 150, "type":"multi-choice"}
+        ] 
+    }
+]

continuous/scenario-12.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "NLVR2","subset_id": 151, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 151, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 151, "type":"multi-choice"},
+            {"dataset": "PatternCom","subset_id": 151, "type":"multi-choice"}
+        ] 
+    }
+]

continuous/scenario-14.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 153, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 153, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 153, "type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 153, "type":"multi-choice"}
+        ] 
+    }
+]

continuous/scenario-18.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 300, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 300, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 300, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 300, "type":"multi-choice"}
+        ] 
+    }
+]

continuous/scenario-5.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 180, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 181, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 182, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 183, "type":"multi-choice"}
+        ] 
+    }
+]

continuous/scenario-8.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "NLVR2","subset_id": 152, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 152, "type":"multi-choice"},
+            {"dataset": "PatternCom","subset_id": 152, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 152, "type":"multi-choice"}
+        ] 
+    }
+]

create_scenario.py

@@ -0,0 +1,42 @@
+import random
+import os
+import argparse
+import json
+
+def main():
+    # Create ArgumentParser object
+    parser = argparse.ArgumentParser()
+
+    # Add the method_name argument
+    parser.add_argument('--seed', type=int, default=0, help='Random seed')
+    parser.add_argument('--client_num', type=int, help='Number of client')
+    parser.add_argument('--max_task', type=int, default=4, help='Maximum number of tasks per client')
+
+    # Parse the command-line arguments
+    args = parser.parse_args()
+    
+    random.seed(args.seed)
+    
+    with open('./all_tasks.json', 'r') as fp:
+        task_list = json.load(fp)
+        
+    print('total available tasks:', len(task_list))
+    
+    max_task_per_client = min(args.max_task, len(task_list) // args.client_num)
+    scenario_data = []
+    
+    random.shuffle(task_list)
+    
+    for client_id in range(args.client_num):
+        scenario_data.append({
+            'client_id':client_id,
+            'datasets':task_list[client_id*max_task_per_client:(client_id+1)*max_task_per_client]
+        })
+    
+    output_path = f'scenario_{args.client_num}clients_{max_task_per_client}tasks_seed{args.seed}.json'
+    
+    with open(output_path, 'w') as json_file:
+        json.dump(scenario_data, json_file, indent=4)
+    
+if __name__ == "__main__":
+    main()

disjoint/scenario-1.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-10.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"}
+        ]
+    }
+]

disjoint/scenario-102.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "DreamSim","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-103.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-104.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "MagicBrush","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-105.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-106.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-107.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Mantis_contrastive_caption","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-11.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-12.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"},
+            {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-14.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-15.json

@@ -0,0 +1,15 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "ScienceQA","subset_id": 4, "type":"onlyLang"},
+            {"dataset": "TextVQA","subset_id": 6, "type":"ocr"},
+            {"dataset": "ImageNet","subset_id": 4, "type":"classification"},
+            {"dataset": "GQA","subset_id": 4, "type":"truth-alignment"},
+            {"dataset": "VizWiz2","subset_id": 4, "type":"truth-alignment"},
+            {"dataset": "COCO2014","subset_id": 4, "type":"grounding"},
+            {"dataset": "COCO2014","subset_id": 5, "type":"truth-alignment"},
+            {"dataset": "OCR-VQA","subset_id": 4, "type":"truth-alignment"}
+        ]
+    }
+]

disjoint/scenario-16.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "M3IT","subset_id": 3, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "MANTIS","subset_id": 2, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "LaMM","subset_id": 2, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "VisionFLAN","subset_id": 2, "type":"open-ended", "metric":"accuracy"}
+        ]
+    }
+]

disjoint/scenario-17.json

@@ -0,0 +1,14 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "NLVR2-2","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB2","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "PatternCom-2","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "dvqa2","subset_id": 99, "type":"multi-choice"},
+            {"dataset": "HQ_Edit2","subset_id": 99, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-18.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-2.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "mPLUG","subset_id": 0, "type":"open-ended"},
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-20.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "ChartQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "DocVQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "iconqa2","subset_id": 0, "type":"open-ended", "metric":"accuracy"},
+            {"dataset": "PathVQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"}
+        ]
+    }
+]

disjoint/scenario-22.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-28.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-3.json

@@ -0,0 +1,11 @@
+[
+{
+    "client_id": 0,
+    "datasets":[
+        {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+        {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"},
+        {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"},
+        {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+    ]
+}
+]

disjoint/scenario-32.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"},
+            {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-38.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-4.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 100, "type":"multi-choice"},
+            {"dataset": "mPLUG","subset_id": 100, "type":"open-ended"},
+            {"dataset": "NLVR2","subset_id": 100,"type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-5.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 51, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 52, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 53, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-6.json

@@ -0,0 +1,8 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 90, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-7.json

@@ -0,0 +1,12 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "WebQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "AQUA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}
+            
+        ] 
+    }
+]

disjoint/scenario-8.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"}
+        ] 
+    }
+]

disjoint/scenario-9.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

fcl_scenarios/scenario-0.json

@@ -0,0 +1,92 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 2, "type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 3, "type":"multi-choice"},
+            {"dataset": "FashionIQ","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "CIRR","subset_id": 1,"type":"multi-choice"},
+            {"dataset": "HQ_Edit","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "PatternCom","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "WebQA","subset_id": 0, "type":"open-ended", "metric":"F1"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "AQUA","subset_id": 0, "type":"open-ended", "metric":"F1"},
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 4,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 5,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "iconqa","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "iconqa","subset_id": 2, "type":"open-ended", "metric":"F1"},
+            {"dataset": "iconqa","subset_id": 3, "type":"open-ended", "metric":"F1"}
+        ]
+    },
+    {
+        "client_id": 6,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 7,
+        "datasets":[
+            {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "IEdit","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 8,
+        "datasets":[
+            {"dataset": "PororoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "VIST","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "AESOP","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 9,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-1.json

@@ -0,0 +1,14 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"},
+            {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"},
+            {"dataset": "ALFRED","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "DiDeMoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"},
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-10.json

@@ -0,0 +1,26 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "DreamSim","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "Mementos","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 8, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-2.json

@@ -0,0 +1,26 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "ALFRED","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "DiDeMoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-20.json

@@ -0,0 +1,92 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"},
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "WebQA","subset_id": 0, "type":"open-ended"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "AQUA","subset_id": 0, "type":"open-ended"},
+            {"dataset": "AQUA","subset_id": 1, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 4,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 5,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 4, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 8, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 6,
+        "datasets":[
+            {"dataset": "mPLUG","subset_id": 0, "type":"open-ended"},
+            {"dataset": "mPLUG","subset_id": 1, "type":"open-ended"},
+            {"dataset": "mPLUG","subset_id": 2, "type":"open-ended"},
+            {"dataset": "mPLUG","subset_id": 4, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 7,
+        "datasets":[
+            {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended"},
+            {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended"},
+            {"dataset": "IEdit","subset_id": 0, "type":"open-ended"},
+            {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 8,
+        "datasets":[
+            {"dataset": "PororoSV","subset_id": 0, "type":"open-ended"},
+            {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended"},
+            {"dataset": "VIST","subset_id": 0, "type":"open-ended"},
+            {"dataset": "AESOP","subset_id": 0, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 9,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-21.json

@@ -0,0 +1,20 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"},
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-22.json

@@ -0,0 +1,38 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"},
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-23.json

@@ -0,0 +1,11 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"}
+        ] 
+    }
+]

fcl_scenarios/scenario-24.json

@@ -0,0 +1,20 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"},
+            {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"},
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-28.json

@@ -0,0 +1,38 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "WebQA","subset_id": 0, "type":"open-ended"},
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "AQUA","subset_id": 0, "type":"open-ended"},
+            {"dataset": "AQUA","subset_id": 1, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended"},
+            {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended"},
+            {"dataset": "IEdit","subset_id": 0, "type":"open-ended"},
+            {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-29.json

@@ -0,0 +1,38 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"},
+            {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"},
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"},
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "PororoSV","subset_id": 0, "type":"open-ended"},
+            {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended"},
+            {"dataset": "VIST","subset_id": 0, "type":"open-ended"},
+            {"dataset": "AESOP","subset_id": 0, "type":"open-ended"}
+        ]
+    }
+]

fcl_scenarios/scenario-3.json

@@ -0,0 +1,20 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "mPLUG","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-4.json

@@ -0,0 +1,14 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "mPLUG","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"}
+        ]
+    }
+]

fcl_scenarios/scenario-40.json

@@ -0,0 +1,242 @@
+[
+    {
+        "client_id": 0,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 1,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 2, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 2,
+        "datasets":[
+            {"dataset": "Fashion200K","subset_id": 3, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 3,
+        "datasets":[
+            {"dataset": "FashionIQ","subset_id": 0, "type":"multi-choice"}
+        ] 
+    },
+    {
+        "client_id": 4,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 5,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 6,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 7,
+        "datasets":[
+            {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 8,
+        "datasets":[
+            {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 9,
+        "datasets":[
+            {"dataset": "CIRR","subset_id": 1,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 10,
+        "datasets":[
+            {"dataset": "HQ_Edit","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 11,
+        "datasets":[
+            {"dataset": "PatternCom","subset_id": 0,"type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 12,
+        "datasets":[
+            {"dataset": "WebQA","subset_id": 0, "type":"open-ended", "metric":"F1"}
+        ]
+    },
+    {
+        "client_id": 13,
+        "datasets":[
+            {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 14,
+        "datasets":[
+            {"dataset": "AQUA","subset_id": 0, "type":"open-ended", "metric":"F1"}
+        ]
+    },
+    {
+        "client_id": 15,
+        "datasets":[
+            {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 16,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 17,
+        "datasets":[
+            {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 18,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 19,
+        "datasets":[
+            {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 20,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 21,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 22,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 2, "type":"open-ended", "metric":"F1"}
+        ]
+    },
+    {
+        "client_id": 23,
+        "datasets":[
+            {"dataset": "iconqa","subset_id": 3, "type":"open-ended", "metric":"F1"}
+        ]
+    },
+    {
+        "client_id": 24,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 25,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 26,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 27,
+        "datasets":[
+            {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 28,
+        "datasets":[
+            {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 29,
+        "datasets":[
+            {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 30,
+        "datasets":[
+            {"dataset": "IEdit","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 31,
+        "datasets":[
+            {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 32,
+        "datasets":[
+            {"dataset": "PororoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 33,
+        "datasets":[
+            {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 34,
+        "datasets":[
+            {"dataset": "VIST","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 35,
+        "datasets":[
+            {"dataset": "AESOP","subset_id": 0, "type":"open-ended", "metric":"RougeL"}
+        ]
+    },
+    {
+        "client_id": 36,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 37,
+        "datasets":[
+            {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 38,
+        "datasets":[
+            {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}
+        ]
+    },
+    {
+        "client_id": 39,
+        "datasets":[
+            {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"}
+        ]
+    }
+]