diff --git a/all_tasks.json b/all_tasks.json new file mode 100644 index 0000000000000000000000000000000000000000..b80e5a6322fd4a3320989bece71a200ba28386c0 --- /dev/null +++ b/all_tasks.json @@ -0,0 +1,34 @@ +[ + {"dataset": "AQUA","subset_id": 0, "task_name": "no external knowledge"}, + {"dataset": "AQUA","subset_id": 1, "task_name": "need external knowledge 1"}, + {"dataset": "AQUA","subset_id": 2, "task_name": "need external knowledge 2"}, + {"dataset": "Mementos","subset_id": 0}, + {"dataset": "SciCap","subset_id": 0, "task_name": "Yes-subfig"}, + {"dataset": "SciCap","subset_id": 1, "task_name": "No-subfig"}, + {"dataset": "Bongard-OpenWorld","subset_id": 0}, + {"dataset": "HRVQA","subset_id": 0, "task_name": "areas"}, + {"dataset": "HRVQA","subset_id": 1, "task_name": "color"}, + {"dataset": "HRVQA","subset_id": 2, "task_name": "location"}, + {"dataset": "HRVQA","subset_id": 3, "task_name": "numbers"}, + {"dataset": "HRVQA","subset_id": 4, "task_name": "scene"}, + {"dataset": "HRVQA","subset_id": 5, "task_name": "shape"}, + {"dataset": "HRVQA","subset_id": 6, "task_name": "size"}, + {"dataset": "HRVQA","subset_id": 7, "task_name": "sports"}, + {"dataset": "HRVQA","subset_id": 8, "task_name": "transportation"}, + {"dataset": "HRVQA","subset_id": 9, "task_name": "yes no"}, + {"dataset": "mPLUG","subset_id": 0, "task_name": "areas"}, + {"dataset": "mPLUG","subset_id": 1, "task_name": "color"}, + {"dataset": "mPLUG","subset_id": 2, "task_name": "location"}, + {"dataset": "mPLUG","subset_id": 3, "task_name": "numbers"}, + {"dataset": "mPLUG","subset_id": 4, "task_name": "scene"}, + {"dataset": "mPLUG","subset_id": 5, "task_name": "shape"}, + {"dataset": "mPLUG","subset_id": 6, "task_name": "size"}, + {"dataset": "mPLUG","subset_id": 7, "task_name": "sports"}, + {"dataset": "Birds-to-Words","subset_id": 0}, + {"dataset": "DiDeMoSV","subset_id": 0}, + {"dataset": "VIST","subset_id": 0}, + {"dataset": "Describe-Diff","subset_id": 0}, + {"dataset": "NLVR2","subset_id": 0}, + {"dataset": "NLVR2","subset_id": 1}, + {"dataset": "KGQA","subset_id": 0} +] \ No newline at end of file diff --git a/continuous/scenario-11.json b/continuous/scenario-11.json new file mode 100644 index 0000000000000000000000000000000000000000..feb8ad2b56f686297f25d834f3ca186be64a0495 --- /dev/null +++ b/continuous/scenario-11.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 150, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 150, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 150, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 150, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/continuous/scenario-12.json b/continuous/scenario-12.json new file mode 100644 index 0000000000000000000000000000000000000000..a99732c57632f67fa5b1d45fdee0ed8fb66d2ef3 --- /dev/null +++ b/continuous/scenario-12.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 151, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 151, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 151, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 151, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/continuous/scenario-14.json b/continuous/scenario-14.json new file mode 100644 index 0000000000000000000000000000000000000000..7c384e368628cbcfbddbfe05c0a95cdc7e13c64a --- /dev/null +++ b/continuous/scenario-14.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 153, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 153, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 153, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 153, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/continuous/scenario-18.json b/continuous/scenario-18.json new file mode 100644 index 0000000000000000000000000000000000000000..3ff698c2a051c1e7ecd249d78483a79058bed184 --- /dev/null +++ b/continuous/scenario-18.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 300, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 300, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 300, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 300, "type":"multi-choice"} + ] + } +] diff --git a/continuous/scenario-5.json b/continuous/scenario-5.json new file mode 100644 index 0000000000000000000000000000000000000000..944fbfa8ab34e20f67acc7f080f9471b721d2352 --- /dev/null +++ b/continuous/scenario-5.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 180, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 181, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 182, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 183, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/continuous/scenario-8.json b/continuous/scenario-8.json new file mode 100644 index 0000000000000000000000000000000000000000..8c7b30f942b316658d9e84b7ca8d5d4e8eb4d08f --- /dev/null +++ b/continuous/scenario-8.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 152, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 152, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 152, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 152, "type":"multi-choice"} + ] + } +] diff --git a/create_scenario.py b/create_scenario.py new file mode 100644 index 0000000000000000000000000000000000000000..c58df121d3d1f1c1abedf384b8a5bf077f517cb1 --- /dev/null +++ b/create_scenario.py @@ -0,0 +1,42 @@ +import random +import os +import argparse +import json + +def main(): + # Create ArgumentParser object + parser = argparse.ArgumentParser() + + # Add the method_name argument + parser.add_argument('--seed', type=int, default=0, help='Random seed') + parser.add_argument('--client_num', type=int, help='Number of client') + parser.add_argument('--max_task', type=int, default=4, help='Maximum number of tasks per client') + + # Parse the command-line arguments + args = parser.parse_args() + + random.seed(args.seed) + + with open('./all_tasks.json', 'r') as fp: + task_list = json.load(fp) + + print('total available tasks:', len(task_list)) + + max_task_per_client = min(args.max_task, len(task_list) // args.client_num) + scenario_data = [] + + random.shuffle(task_list) + + for client_id in range(args.client_num): + scenario_data.append({ + 'client_id':client_id, + 'datasets':task_list[client_id*max_task_per_client:(client_id+1)*max_task_per_client] + }) + + output_path = f'scenario_{args.client_num}clients_{max_task_per_client}tasks_seed{args.seed}.json' + + with open(output_path, 'w') as json_file: + json.dump(scenario_data, json_file, indent=4) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/disjoint/scenario-1.json b/disjoint/scenario-1.json new file mode 100644 index 0000000000000000000000000000000000000000..e44d2cc5f4eec24b1ae10f1a0c008d8c62f3ea32 --- /dev/null +++ b/disjoint/scenario-1.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-10.json b/disjoint/scenario-10.json new file mode 100644 index 0000000000000000000000000000000000000000..c96d4a09349ac896bdd0922c27c99ef25057d405 --- /dev/null +++ b/disjoint/scenario-10.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-102.json b/disjoint/scenario-102.json new file mode 100644 index 0000000000000000000000000000000000000000..d377f2631d72e3093eb6e953f685719306734545 --- /dev/null +++ b/disjoint/scenario-102.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "DreamSim","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-103.json b/disjoint/scenario-103.json new file mode 100644 index 0000000000000000000000000000000000000000..b453c6a40915e17e19bb72454a7c3f2b23362b3d --- /dev/null +++ b/disjoint/scenario-103.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-104.json b/disjoint/scenario-104.json new file mode 100644 index 0000000000000000000000000000000000000000..bffdc44d230a9ee6467a831e5e1876dd55af39ae --- /dev/null +++ b/disjoint/scenario-104.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-105.json b/disjoint/scenario-105.json new file mode 100644 index 0000000000000000000000000000000000000000..111d88b787362624762a12240eb38ce9a597fb7b --- /dev/null +++ b/disjoint/scenario-105.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-106.json b/disjoint/scenario-106.json new file mode 100644 index 0000000000000000000000000000000000000000..88567d158afb42b5a99239ccc58388428e3e5b64 --- /dev/null +++ b/disjoint/scenario-106.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-107.json b/disjoint/scenario-107.json new file mode 100644 index 0000000000000000000000000000000000000000..cb11150c30c8cd6300a8ff75f1dd605c0de0b9dd --- /dev/null +++ b/disjoint/scenario-107.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Mantis_contrastive_caption","subset_id": 0, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-11.json b/disjoint/scenario-11.json new file mode 100644 index 0000000000000000000000000000000000000000..0871a999bcdc666db2e9931a71d3979f21ea4b8e --- /dev/null +++ b/disjoint/scenario-11.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-12.json b/disjoint/scenario-12.json new file mode 100644 index 0000000000000000000000000000000000000000..673f994314e14c764dc5948c6f59d8cc72d80199 --- /dev/null +++ b/disjoint/scenario-12.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"}, + {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-14.json b/disjoint/scenario-14.json new file mode 100644 index 0000000000000000000000000000000000000000..9515b2b50da59db0ec677b79df33aae76bb283eb --- /dev/null +++ b/disjoint/scenario-14.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-15.json b/disjoint/scenario-15.json new file mode 100644 index 0000000000000000000000000000000000000000..8d48e19237eda1737121866e7754d28eebe687fd --- /dev/null +++ b/disjoint/scenario-15.json @@ -0,0 +1,15 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "ScienceQA","subset_id": 4, "type":"onlyLang"}, + {"dataset": "TextVQA","subset_id": 6, "type":"ocr"}, + {"dataset": "ImageNet","subset_id": 4, "type":"classification"}, + {"dataset": "GQA","subset_id": 4, "type":"truth-alignment"}, + {"dataset": "VizWiz2","subset_id": 4, "type":"truth-alignment"}, + {"dataset": "COCO2014","subset_id": 4, "type":"grounding"}, + {"dataset": "COCO2014","subset_id": 5, "type":"truth-alignment"}, + {"dataset": "OCR-VQA","subset_id": 4, "type":"truth-alignment"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-16.json b/disjoint/scenario-16.json new file mode 100644 index 0000000000000000000000000000000000000000..aad706e478b8460ba09d09e3157d29eea7368ca4 --- /dev/null +++ b/disjoint/scenario-16.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "M3IT","subset_id": 3, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "MANTIS","subset_id": 2, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "LaMM","subset_id": 2, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "VisionFLAN","subset_id": 2, "type":"open-ended", "metric":"accuracy"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-17.json b/disjoint/scenario-17.json new file mode 100644 index 0000000000000000000000000000000000000000..dec2e8bc41b105579a4e3a82f2612727919cd010 --- /dev/null +++ b/disjoint/scenario-17.json @@ -0,0 +1,14 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 99, "type":"multi-choice"}, + {"dataset": "NLVR2-2","subset_id": 99, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB2","subset_id": 99, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 99, "type":"multi-choice"}, + {"dataset": "PatternCom-2","subset_id": 99, "type":"multi-choice"}, + {"dataset": "dvqa2","subset_id": 99, "type":"multi-choice"}, + {"dataset": "HQ_Edit2","subset_id": 99, "type":"multi-choice"} + ] + } +] diff --git a/disjoint/scenario-18.json b/disjoint/scenario-18.json new file mode 100644 index 0000000000000000000000000000000000000000..841505f4ddc56ecc59fa33761ec4bea05b3dbf72 --- /dev/null +++ b/disjoint/scenario-18.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-2.json b/disjoint/scenario-2.json new file mode 100644 index 0000000000000000000000000000000000000000..d943931031e89890a5f8150f29537473a9c5e3e1 --- /dev/null +++ b/disjoint/scenario-2.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}, + {"dataset": "mPLUG","subset_id": 0, "type":"open-ended"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-20.json b/disjoint/scenario-20.json new file mode 100644 index 0000000000000000000000000000000000000000..5bb2e366bad0ed3fea39b2e76e31c07185435167 --- /dev/null +++ b/disjoint/scenario-20.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "ChartQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "DocVQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "iconqa2","subset_id": 0, "type":"open-ended", "metric":"accuracy"}, + {"dataset": "PathVQA2","subset_id": 0, "type":"open-ended", "metric":"accuracy"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-22.json b/disjoint/scenario-22.json new file mode 100644 index 0000000000000000000000000000000000000000..0a7c9ee33d45e7280be1ed885297f8360a66ec85 --- /dev/null +++ b/disjoint/scenario-22.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-28.json b/disjoint/scenario-28.json new file mode 100644 index 0000000000000000000000000000000000000000..49c051c9a6ac2726fab89b8991c7b4e8ebd001fa --- /dev/null +++ b/disjoint/scenario-28.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-3.json b/disjoint/scenario-3.json new file mode 100644 index 0000000000000000000000000000000000000000..728c4dbbf16baf9b52d49e2b3b906a7e2b415915 --- /dev/null +++ b/disjoint/scenario-3.json @@ -0,0 +1,11 @@ +[ +{ + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] +} +] \ No newline at end of file diff --git a/disjoint/scenario-32.json b/disjoint/scenario-32.json new file mode 100644 index 0000000000000000000000000000000000000000..b14f48e4e2c56d6d8835514e5a8beec48c06508a --- /dev/null +++ b/disjoint/scenario-32.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HQ_Edit-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom-2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI2","subset_id": 50, "type":"multi-choice"}, + {"dataset": "NLVR2-2","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-38.json b/disjoint/scenario-38.json new file mode 100644 index 0000000000000000000000000000000000000000..a66bb055b745508e0ecfc6f3d92bdb6972a0e8dc --- /dev/null +++ b/disjoint/scenario-38.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-4.json b/disjoint/scenario-4.json new file mode 100644 index 0000000000000000000000000000000000000000..4e7ec52449d9981d98b7cd96e2ce0fd27a74bf11 --- /dev/null +++ b/disjoint/scenario-4.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 100, "type":"multi-choice"}, + {"dataset": "mPLUG","subset_id": 100, "type":"open-ended"}, + {"dataset": "NLVR2","subset_id": 100,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-5.json b/disjoint/scenario-5.json new file mode 100644 index 0000000000000000000000000000000000000000..f76e68592b8ef38028fed4770df1a9101e9fda41 --- /dev/null +++ b/disjoint/scenario-5.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 51, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 52, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 53, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-6.json b/disjoint/scenario-6.json new file mode 100644 index 0000000000000000000000000000000000000000..14b3e0aabbce85c5d4f2d47fc116b988084ce589 --- /dev/null +++ b/disjoint/scenario-6.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 90, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-7.json b/disjoint/scenario-7.json new file mode 100644 index 0000000000000000000000000000000000000000..f54e306e41bbfb9ae7a8a850d8c3c98a34b7d75e --- /dev/null +++ b/disjoint/scenario-7.json @@ -0,0 +1,12 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "AQUA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"} + + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-8.json b/disjoint/scenario-8.json new file mode 100644 index 0000000000000000000000000000000000000000..b902e30d0f998c94848d3d34f133cfa4767abeb7 --- /dev/null +++ b/disjoint/scenario-8.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/disjoint/scenario-9.json b/disjoint/scenario-9.json new file mode 100644 index 0000000000000000000000000000000000000000..7b1668860d160ab8e99ffc573d04df11c513856a --- /dev/null +++ b/disjoint/scenario-9.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-0.json b/fcl_scenarios/scenario-0.json new file mode 100644 index 0000000000000000000000000000000000000000..6c0412e2f5e4b0f0473910837d706cc2de03c387 --- /dev/null +++ b/fcl_scenarios/scenario-0.json @@ -0,0 +1,92 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 2, "type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 3, "type":"multi-choice"}, + {"dataset": "FashionIQ","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"}, + {"dataset": "CIRR","subset_id": 1,"type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0,"type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"open-ended", "metric":"F1"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "AQUA","subset_id": 0, "type":"open-ended", "metric":"F1"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 4, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"}, + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 5, + "datasets":[ + {"dataset": "iconqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "iconqa","subset_id": 1, "type":"multi-choice"}, + {"dataset": "iconqa","subset_id": 2, "type":"open-ended", "metric":"F1"}, + {"dataset": "iconqa","subset_id": 3, "type":"open-ended", "metric":"F1"} + ] + }, + { + "client_id": 6, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 7, + "datasets":[ + {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "IEdit","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 8, + "datasets":[ + {"dataset": "PororoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "VIST","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "AESOP","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 9, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-1.json b/fcl_scenarios/scenario-1.json new file mode 100644 index 0000000000000000000000000000000000000000..785f74ba3bc51d29f150e16312c37d7376176d96 --- /dev/null +++ b/fcl_scenarios/scenario-1.json @@ -0,0 +1,14 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"}, + {"dataset": "ALFRED","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "DiDeMoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}, + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-10.json b/fcl_scenarios/scenario-10.json new file mode 100644 index 0000000000000000000000000000000000000000..b9c673717cc9eab4fa6b065a700036ab7ef232dd --- /dev/null +++ b/fcl_scenarios/scenario-10.json @@ -0,0 +1,26 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "DreamSim","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "Mementos","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 8, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-2.json b/fcl_scenarios/scenario-2.json new file mode 100644 index 0000000000000000000000000000000000000000..57e8a513606702d8ebecad432beab038d2f83f2b --- /dev/null +++ b/fcl_scenarios/scenario-2.json @@ -0,0 +1,26 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "ALFRED","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "DiDeMoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-20.json b/fcl_scenarios/scenario-20.json new file mode 100644 index 0000000000000000000000000000000000000000..aaaf32270e93ced9edfb22d0c195a0d269f94159 --- /dev/null +++ b/fcl_scenarios/scenario-20.json @@ -0,0 +1,92 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"}, + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"open-ended"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "AQUA","subset_id": 0, "type":"open-ended"}, + {"dataset": "AQUA","subset_id": 1, "type":"open-ended"} + ] + }, + { + "client_id": 4, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"}, + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 5, + "datasets":[ + {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 4, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 8, "type":"multi-choice"} + ] + }, + { + "client_id": 6, + "datasets":[ + {"dataset": "mPLUG","subset_id": 0, "type":"open-ended"}, + {"dataset": "mPLUG","subset_id": 1, "type":"open-ended"}, + {"dataset": "mPLUG","subset_id": 2, "type":"open-ended"}, + {"dataset": "mPLUG","subset_id": 4, "type":"open-ended"} + ] + }, + { + "client_id": 7, + "datasets":[ + {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended"}, + {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended"}, + {"dataset": "IEdit","subset_id": 0, "type":"open-ended"}, + {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended"} + ] + }, + { + "client_id": 8, + "datasets":[ + {"dataset": "PororoSV","subset_id": 0, "type":"open-ended"}, + {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended"}, + {"dataset": "VIST","subset_id": 0, "type":"open-ended"}, + {"dataset": "AESOP","subset_id": 0, "type":"open-ended"} + ] + }, + { + "client_id": 9, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-21.json b/fcl_scenarios/scenario-21.json new file mode 100644 index 0000000000000000000000000000000000000000..ea2a262e6ce29b148b62d59826795237b3d3adef --- /dev/null +++ b/fcl_scenarios/scenario-21.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"}, + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-22.json b/fcl_scenarios/scenario-22.json new file mode 100644 index 0000000000000000000000000000000000000000..9fe59fc14acdedd1ff95b6f59eee07b04ce60141 --- /dev/null +++ b/fcl_scenarios/scenario-22.json @@ -0,0 +1,38 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"}, + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"}, + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-23.json b/fcl_scenarios/scenario-23.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e60a7667eada8c0211a499681e3600201ffbda --- /dev/null +++ b/fcl_scenarios/scenario-23.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-24.json b/fcl_scenarios/scenario-24.json new file mode 100644 index 0000000000000000000000000000000000000000..ea2a262e6ce29b148b62d59826795237b3d3adef --- /dev/null +++ b/fcl_scenarios/scenario-24.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0,"type":"multi-choice"}, + {"dataset": "Fashion200K","subset_id": 1,"type":"multi-choice"}, + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-28.json b/fcl_scenarios/scenario-28.json new file mode 100644 index 0000000000000000000000000000000000000000..8b62b368b2de69dc632372dbcd5b3d7167fa3dff --- /dev/null +++ b/fcl_scenarios/scenario-28.json @@ -0,0 +1,38 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"open-ended"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "AQUA","subset_id": 0, "type":"open-ended"}, + {"dataset": "AQUA","subset_id": 1, "type":"open-ended"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended"}, + {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended"}, + {"dataset": "IEdit","subset_id": 0, "type":"open-ended"}, + {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"}, + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"}, + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-29.json b/fcl_scenarios/scenario-29.json new file mode 100644 index 0000000000000000000000000000000000000000..5509bd6c81311f23c63b5b0c97d82edbb911a2fe --- /dev/null +++ b/fcl_scenarios/scenario-29.json @@ -0,0 +1,38 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "RecipeQA_VisualCloze","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Dialogue","subset_id": 0, "type":"multi-choice"}, + {"dataset": "COMICS_Panel","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 0, "type":"open-ended"}, + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "PororoSV","subset_id": 0, "type":"open-ended"}, + {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended"}, + {"dataset": "VIST","subset_id": 0, "type":"open-ended"}, + {"dataset": "AESOP","subset_id": 0, "type":"open-ended"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-3.json b/fcl_scenarios/scenario-3.json new file mode 100644 index 0000000000000000000000000000000000000000..af2490172dc9c0766e4dc6c67b3e1e4851827b70 --- /dev/null +++ b/fcl_scenarios/scenario-3.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "mPLUG","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-4.json b/fcl_scenarios/scenario-4.json new file mode 100644 index 0000000000000000000000000000000000000000..68c6075796b013b6fcfaae58978f6fae0420d13f --- /dev/null +++ b/fcl_scenarios/scenario-4.json @@ -0,0 +1,14 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "mPLUG","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "iconqa","subset_id": 4, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-40.json b/fcl_scenarios/scenario-40.json new file mode 100644 index 0000000000000000000000000000000000000000..982d6f7cf6cabbd023b0bf8ec97bde942143da83 --- /dev/null +++ b/fcl_scenarios/scenario-40.json @@ -0,0 +1,242 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 2, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Fashion200K","subset_id": 3, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "FashionIQ","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 4, + "datasets":[ + {"dataset": "HRVQA","subset_id": 2, "type":"multi-choice"} + ] + }, + { + "client_id": 5, + "datasets":[ + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"} + ] + }, + { + "client_id": 6, + "datasets":[ + {"dataset": "HRVQA","subset_id": 7, "type":"multi-choice"} + ] + }, + { + "client_id": 7, + "datasets":[ + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] + }, + { + "client_id": 8, + "datasets":[ + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 9, + "datasets":[ + {"dataset": "CIRR","subset_id": 1,"type":"multi-choice"} + ] + }, + { + "client_id": 10, + "datasets":[ + {"dataset": "HQ_Edit","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 11, + "datasets":[ + {"dataset": "PatternCom","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 12, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"open-ended", "metric":"F1"} + ] + }, + { + "client_id": 13, + "datasets":[ + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 14, + "datasets":[ + {"dataset": "AQUA","subset_id": 0, "type":"open-ended", "metric":"F1"} + ] + }, + { + "client_id": 15, + "datasets":[ + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 16, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 3, "type":"multi-choice"} + ] + }, + { + "client_id": 17, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 18, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 19, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 20, + "datasets":[ + {"dataset": "iconqa","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 21, + "datasets":[ + {"dataset": "iconqa","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 22, + "datasets":[ + {"dataset": "iconqa","subset_id": 2, "type":"open-ended", "metric":"F1"} + ] + }, + { + "client_id": 23, + "datasets":[ + {"dataset": "iconqa","subset_id": 3, "type":"open-ended", "metric":"F1"} + ] + }, + { + "client_id": 24, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 25, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 26, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 27, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 28, + "datasets":[ + {"dataset": "Spot-the-Diff","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 29, + "datasets":[ + {"dataset": "Birds-to-Words","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 30, + "datasets":[ + {"dataset": "IEdit","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 31, + "datasets":[ + {"dataset": "CLEVR-Change","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 32, + "datasets":[ + {"dataset": "PororoSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 33, + "datasets":[ + {"dataset": "FlintstonesSV","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 34, + "datasets":[ + {"dataset": "VIST","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 35, + "datasets":[ + {"dataset": "AESOP","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 36, + "datasets":[ + {"dataset": "VISION","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 37, + "datasets":[ + {"dataset": "VISION","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 38, + "datasets":[ + {"dataset": "VizWiz","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 39, + "datasets":[ + {"dataset": "MIT-States","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-5.json b/fcl_scenarios/scenario-5.json new file mode 100644 index 0000000000000000000000000000000000000000..b40c918486d1cbe353da58ba5151a12e4e383789 --- /dev/null +++ b/fcl_scenarios/scenario-5.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 2, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-6.json b/fcl_scenarios/scenario-6.json new file mode 100644 index 0000000000000000000000000000000000000000..4bf2a40c44ed965c27e1c4dd38da291aac3290d2 --- /dev/null +++ b/fcl_scenarios/scenario-6.json @@ -0,0 +1,26 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 5, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 6, "type":"multi-choice"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 7, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 8, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-7.json b/fcl_scenarios/scenario-7.json new file mode 100644 index 0000000000000000000000000000000000000000..677cd89a07f6b0640bd28a389f3b3394bf799fee --- /dev/null +++ b/fcl_scenarios/scenario-7.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "DreamSim","subset_id": 0, "type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "Mementos","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-8.json b/fcl_scenarios/scenario-8.json new file mode 100644 index 0000000000000000000000000000000000000000..a639a2898337491cbd2304cd303c72fc46e94e81 --- /dev/null +++ b/fcl_scenarios/scenario-8.json @@ -0,0 +1,26 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 1, "type":"multi-choice"} + ] + }, + { + "client_id": 3, + "datasets":[ + {"dataset": "RecipeQA_TextCloze","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/fcl_scenarios/scenario-9.json b/fcl_scenarios/scenario-9.json new file mode 100644 index 0000000000000000000000000000000000000000..286e70429eae2ca9c22bda16a8dc169ad8108b4b --- /dev/null +++ b/fcl_scenarios/scenario-9.json @@ -0,0 +1,20 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "RecipeQA_ImageCoherence","subset_id": 0,"type":"multi-choice"} + ] + }, + { + "client_id": 1, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 0, "type":"open-ended", "metric":"RougeL"} + ] + }, + { + "client_id": 2, + "datasets":[ + {"dataset": "MagicBrush","subset_id": 1, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/periodic/scenario-11.json b/periodic/scenario-11.json new file mode 100644 index 0000000000000000000000000000000000000000..7599082df2ff78b8801133d1311185233b5c8c04 --- /dev/null +++ b/periodic/scenario-11.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 160, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 160, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 160, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 160, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/periodic/scenario-12.json b/periodic/scenario-12.json new file mode 100644 index 0000000000000000000000000000000000000000..0b52c61fecef1e36d3f10def7a77f31987c5b1a9 --- /dev/null +++ b/periodic/scenario-12.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 161, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 161, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 161, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 161, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/periodic/scenario-14.json b/periodic/scenario-14.json new file mode 100644 index 0000000000000000000000000000000000000000..44dc5e800921a8a85ad9bd5d06b1c9a8ede165a8 --- /dev/null +++ b/periodic/scenario-14.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 163, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 163, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 163, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 163, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/periodic/scenario-18.json b/periodic/scenario-18.json new file mode 100644 index 0000000000000000000000000000000000000000..8680ed802e2bf4161caaaf452a1313434ce2350a --- /dev/null +++ b/periodic/scenario-18.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 301, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 301, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 301, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 301, "type":"multi-choice"} + ] + } +] diff --git a/periodic/scenario-5.json b/periodic/scenario-5.json new file mode 100644 index 0000000000000000000000000000000000000000..4bc420bdfe23d52623cdb8f58cebf861c97e03b2 --- /dev/null +++ b/periodic/scenario-5.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 190, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 191, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 192, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 193, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/periodic/scenario-8.json b/periodic/scenario-8.json new file mode 100644 index 0000000000000000000000000000000000000000..efd4a12c86fc9d0ef735665baf8edcd1f2e3ffa4 --- /dev/null +++ b/periodic/scenario-8.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 162, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 162, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 162, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 162, "type":"multi-choice"} + ] + } +] diff --git a/scenario-1.json b/scenario-1.json new file mode 100644 index 0000000000000000000000000000000000000000..e44d2cc5f4eec24b1ae10f1a0c008d8c62f3ea32 --- /dev/null +++ b/scenario-1.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-10.json b/scenario-10.json new file mode 100644 index 0000000000000000000000000000000000000000..c96d4a09349ac896bdd0922c27c99ef25057d405 --- /dev/null +++ b/scenario-10.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 2, "type":"open-ended", "metric":"RougeL"}, + {"dataset": "Co-Instruct-DB","subset_id": 3, "type":"open-ended", "metric":"RougeL"} + ] + } +] \ No newline at end of file diff --git a/scenario-11.json b/scenario-11.json new file mode 100644 index 0000000000000000000000000000000000000000..0871a999bcdc666db2e9931a71d3979f21ea4b8e --- /dev/null +++ b/scenario-11.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-12.json b/scenario-12.json new file mode 100644 index 0000000000000000000000000000000000000000..36c77a5735cfc9cf96fb6b104b39efba80973bf8 --- /dev/null +++ b/scenario-12.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-14.json b/scenario-14.json new file mode 100644 index 0000000000000000000000000000000000000000..3ed50bd16fb9a2e5307adbb09d4c64cf4de3bf28 --- /dev/null +++ b/scenario-14.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-OpenWorld","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Co-Instruct-DB","subset_id": 0, "type":"multi-choice"}, + {"dataset": "dqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-15.json b/scenario-15.json new file mode 100644 index 0000000000000000000000000000000000000000..9787d41fbc9bb7c61d47fc49e8198e1ce42be158 --- /dev/null +++ b/scenario-15.json @@ -0,0 +1,15 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "ScienceQA","subset_id": 4, "type":"multi-choice"}, + {"dataset": "TextVQA","subset_id": 4, "type":"multi-choice"}, + {"dataset": "ImageNet","subset_id": 4, "type":"multi-choice"}, + {"dataset": "GQA","subset_id": 4, "type":"multi-choice"}, + {"dataset": "VizWiz","subset_id": 4, "type":"multi-choice"}, + {"dataset": "Grounding","subset_id": 4, "type":"multi-choice"}, + {"dataset": "VQAv2","subset_id": 4, "type":"multi-choice"}, + {"dataset": "OCR-VQA","subset_id": 4, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-2.json b/scenario-2.json new file mode 100644 index 0000000000000000000000000000000000000000..d943931031e89890a5f8150f29537473a9c5e3e1 --- /dev/null +++ b/scenario-2.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 1, "type":"multi-choice"}, + {"dataset": "mPLUG","subset_id": 0, "type":"open-ended"}, + {"dataset": "NLVR2","subset_id": 0,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-3.json b/scenario-3.json new file mode 100644 index 0000000000000000000000000000000000000000..728c4dbbf16baf9b52d49e2b3b906a7e2b415915 --- /dev/null +++ b/scenario-3.json @@ -0,0 +1,11 @@ +[ +{ + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 1, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 5, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 6, "type":"multi-choice"}, + {"dataset": "HRVQA","subset_id": 9, "type":"multi-choice"} + ] +} +] \ No newline at end of file diff --git a/scenario-4.json b/scenario-4.json new file mode 100644 index 0000000000000000000000000000000000000000..4e7ec52449d9981d98b7cd96e2ce0fd27a74bf11 --- /dev/null +++ b/scenario-4.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "HRVQA","subset_id": 100, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 100, "type":"multi-choice"}, + {"dataset": "mPLUG","subset_id": 100, "type":"open-ended"}, + {"dataset": "NLVR2","subset_id": 100,"type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-5.json b/scenario-5.json new file mode 100644 index 0000000000000000000000000000000000000000..f76e68592b8ef38028fed4770df1a9101e9fda41 --- /dev/null +++ b/scenario-5.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 50, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 51, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 52, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 53, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-6.json b/scenario-6.json new file mode 100644 index 0000000000000000000000000000000000000000..14b3e0aabbce85c5d4f2d47fc116b988084ce589 --- /dev/null +++ b/scenario-6.json @@ -0,0 +1,8 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "Bongard-HOI","subset_id": 90, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-7.json b/scenario-7.json new file mode 100644 index 0000000000000000000000000000000000000000..f54e306e41bbfb9ae7a8a850d8c3c98a34b7d75e --- /dev/null +++ b/scenario-7.json @@ -0,0 +1,12 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "WebQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "AQUA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"} + + ] + } +] \ No newline at end of file diff --git a/scenario-8.json b/scenario-8.json new file mode 100644 index 0000000000000000000000000000000000000000..b902e30d0f998c94848d3d34f133cfa4767abeb7 --- /dev/null +++ b/scenario-8.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "NLVR2","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"}, + {"dataset": "Bongard-HOI","subset_id": 2, "type":"multi-choice"} + ] + } +] \ No newline at end of file diff --git a/scenario-9.json b/scenario-9.json new file mode 100644 index 0000000000000000000000000000000000000000..7b1668860d160ab8e99ffc573d04df11c513856a --- /dev/null +++ b/scenario-9.json @@ -0,0 +1,11 @@ +[ + { + "client_id": 0, + "datasets":[ + {"dataset": "dvqa","subset_id": 0, "type":"multi-choice"}, + {"dataset": "HQ_Edit","subset_id": 0, "type":"multi-choice"}, + {"dataset": "TQA","subset_id": 0, "type":"multi-choice"}, + {"dataset": "PatternCom","subset_id": 0, "type":"multi-choice"} + ] + } +] \ No newline at end of file