Duplicate from quantumiracle-git/robotinder

Co-authored-by: Zihan Ding <[email protected]>

.gitattributes

@@ -0,0 +1,82 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+videos/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+videos/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+videos/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+videos/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-4.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-5.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-6.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-7.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-8.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandBlockStack/rl-video-episode-9.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-4.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-5.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-6.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-7.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-8.gif filter=lfs diff=lfs merge=lfs -text
+videos/ShadowHandDoorOpenInward/rl-video-episode-9.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-4.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-5.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-6.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-7.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-8.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandBlockStack/rl-video-episode-9.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-0.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-1.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-2.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-3.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-4.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-5.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-6.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-7.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-8.gif filter=lfs diff=lfs merge=lfs -text
+robotinder-data/ShadowHandDoorOpenInward/rl-video-episode-9.gif filter=lfs diff=lfs merge=lfs -text

Bidexhands_Video.csv

@@ -0,0 +1,241 @@
+env_name,seed,succeed_iteration,comment
+ShadowHand,3,10000,
+ShadowHand,4,10000,
+ShadowHand,5,19000,
+ShadowHand,6,16000,
+ShadowHand,7,17000,
+ShadowHand,8,10000,
+ShadowHand,9,19000,
+ShadowHand,10,12000-18000,
+ShadowHand,11,12000,
+ShadowHand,12,14000-18000,
+ShadowHand,13,11000-18000,
+ShadowHand,14,15000,
+ShadowHandBlockStack,3,18000,
+ShadowHandBlockStack,4,17000,
+ShadowHandBlockStack,5,18000,
+ShadowHandBlockStack,6,unsolved (must >20000),
+ShadowHandBlockStack,7,unsolved (must >20000),
+ShadowHandBlockStack,8,10000,
+ShadowHandBlockStack,9,unsolved (must >20000),
+ShadowHandBlockStack,10,11000,
+ShadowHandBlockStack,11,unsolved (must >20000),
+ShadowHandBlockStack,12,14000,
+ShadowHandBlockStack,13,unsolved (must >20000),
+ShadowHandBlockStack,14,14000,
+ShadowHandBottleCap,3,unsolved (must >20000),
+ShadowHandBottleCap,4,17000,
+ShadowHandBottleCap,5,12000,
+ShadowHandBottleCap,6,18000,
+ShadowHandBottleCap,7,14000,
+ShadowHandBottleCap,8,17000,
+ShadowHandBottleCap,9,11000,
+ShadowHandBottleCap,10,18000,
+ShadowHandBottleCap,11,15000,
+ShadowHandBottleCap,12,19000,
+ShadowHandBottleCap,13,11000,
+ShadowHandBottleCap,14,10000,
+ShadowHandCatchAbreast,3,10000,
+ShadowHandCatchAbreast,4,10000,
+ShadowHandCatchAbreast,5,10000,
+ShadowHandCatchAbreast,6,10000,
+ShadowHandCatchAbreast,7,10000,
+ShadowHandCatchAbreast,8,10000,
+ShadowHandCatchAbreast,9,10000,
+ShadowHandCatchAbreast,10,10000,
+ShadowHandCatchAbreast,11,10000-17000,
+ShadowHandCatchAbreast,12,12000,
+ShadowHandCatchAbreast,13,12000,
+ShadowHandCatchAbreast,14,10000,
+ShadowHandCatchOver2Underarm,3,10000,
+ShadowHandCatchOver2Underarm,4,10000,
+ShadowHandCatchOver2Underarm,5,10000,
+ShadowHandCatchOver2Underarm,6,10000,
+ShadowHandCatchOver2Underarm,7,10000,
+ShadowHandCatchOver2Underarm,8,10000,
+ShadowHandCatchOver2Underarm,9,10000,
+ShadowHandCatchOver2Underarm,10,13000,
+ShadowHandCatchOver2Underarm,11,18000,(actually unsolved)
+ShadowHandCatchOver2Underarm,12,15000,
+ShadowHandCatchOver2Underarm,13,10000,
+ShadowHandCatchOver2Underarm,14,10000,
+ShadowHandCatchUnderarm,3,10000,
+ShadowHandCatchUnderarm,4,10000,
+ShadowHandCatchUnderarm,5,10000,
+ShadowHandCatchUnderarm,6,10000,
+ShadowHandCatchUnderarm,7,10000,
+ShadowHandCatchUnderarm,8,10000,
+ShadowHandCatchUnderarm,9,10000,
+ShadowHandCatchUnderarm,10,10000,
+ShadowHandCatchUnderarm,11,20000,
+ShadowHandCatchUnderarm,12,10000,
+ShadowHandCatchUnderarm,13,10000,
+ShadowHandCatchUnderarm,14,10000,
+ShadowHandDoorCloseInward,3,10000,
+ShadowHandDoorCloseInward,4,unsolved (must >20000),
+ShadowHandDoorCloseInward,5,unsolved (must >20000),
+ShadowHandDoorCloseInward,6,10000,
+ShadowHandDoorCloseInward,7,unsolved (must >20000),
+ShadowHandDoorCloseInward,8,unsolved (must >20000),
+ShadowHandDoorCloseInward,9,10000,
+ShadowHandDoorCloseInward,10,10000,
+ShadowHandDoorCloseInward,11,10000,
+ShadowHandDoorCloseInward,12,10000,
+ShadowHandDoorCloseInward,13,10000,
+ShadowHandDoorCloseInward,14,unsolved (must >20000),
+ShadowHandDoorCloseOutward,3,20000,
+ShadowHandDoorCloseOutward,4,unsolved task,
+ShadowHandDoorCloseOutward,5,unsolved task,
+ShadowHandDoorCloseOutward,6,unsolved task,
+ShadowHandDoorCloseOutward,7,unsolved task,
+ShadowHandDoorCloseOutward,8,unsolved task,
+ShadowHandDoorCloseOutward,9,unsolved task,
+ShadowHandDoorCloseOutward,10,unsolved task,
+ShadowHandDoorCloseOutward,11,unsolved task,
+ShadowHandDoorCloseOutward,12,unsolved task,
+ShadowHandDoorCloseOutward,13,unsolved task,
+ShadowHandDoorCloseOutward,14,unsolved task,
+ShadowHandDoorOpenOutward,3,10000,
+ShadowHandDoorOpenOutward,4,10000,
+ShadowHandDoorOpenOutward,5,10000,
+ShadowHandDoorOpenOutward,6,10000,
+ShadowHandDoorOpenOutward,7,10000,
+ShadowHandDoorOpenOutward,8,10000,
+ShadowHandDoorOpenOutward,9,10000,
+ShadowHandDoorOpenOutward,10,10000-10000,
+ShadowHandDoorOpenOutward,11,unsolved,
+ShadowHandDoorOpenOutward,12,unsolved,10000 (solve one hand)
+ShadowHandDoorOpenOutward,13,unsolved,11000 (solve one hand)
+ShadowHandDoorOpenOutward,14,10000,
+ShadowHandDoorOpenInward,3,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,4,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,5,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,6,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,7,14000,
+ShadowHandDoorOpenInward,8,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,9,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandDoorOpenInward,10,10000,
+ShadowHandDoorOpenInward,11,unsolved (must >20000),
+ShadowHandDoorOpenInward,12,10000,
+ShadowHandDoorOpenInward,13,10000,
+ShadowHandDoorOpenInward,14,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandGraspAndPlace,3,unsolved task (can solve),
+ShadowHandGraspAndPlace,4,unsolved task (can solve),
+ShadowHandGraspAndPlace,5,10000,
+ShadowHandGraspAndPlace,6,10000,
+ShadowHandGraspAndPlace,7,10000,
+ShadowHandGraspAndPlace,8,18000,
+ShadowHandGraspAndPlace,9,13000,
+ShadowHandGraspAndPlace,10,unsolved ,
+ShadowHandGraspAndPlace,11,unsolved ,
+ShadowHandGraspAndPlace,12,unsolved ,
+ShadowHandGraspAndPlace,13,unsolved ,
+ShadowHandGraspAndPlace,14,unsolved (must >20000),
+ShadowHandKettle,3,20000,
+ShadowHandKettle,4,unsolved task (hard to solve),
+ShadowHandKettle,5,unsolved task (hard to solve),
+ShadowHandKettle,6,unsolved task (hard to solve),
+ShadowHandKettle,7,unsolved task (hard to solve),
+ShadowHandKettle,8,unsolved task (hard to solve),
+ShadowHandKettle,9,unsolved task (hard to solve),
+ShadowHandKettle,10,unsolved task (hard to solve),
+ShadowHandKettle,11,unsolved task (hard to solve),
+ShadowHandKettle,12,unsolved task (hard to solve),
+ShadowHandKettle,13,unsolved task (hard to solve),
+ShadowHandKettle,14,unsolved task (hard to solve),
+ShadowHandLiftUnderarm,3,unsolved (must >20000),
+ShadowHandLiftUnderarm,4,16000,
+ShadowHandLiftUnderarm,5,,
+ShadowHandLiftUnderarm,6,,
+ShadowHandLiftUnderarm,7,,
+ShadowHandLiftUnderarm,8,,
+ShadowHandLiftUnderarm,9,,
+ShadowHandLiftUnderarm,10,10000,
+ShadowHandLiftUnderarm,11,10000,
+ShadowHandLiftUnderarm,12,18000,
+ShadowHandLiftUnderarm,13,10000,
+ShadowHandLiftUnderarm,14,,
+ShadowHandOver,3,10000,
+ShadowHandOver,4,10000,
+ShadowHandOver,5,10000,
+ShadowHandOver,6,10000,
+ShadowHandOver,7,10000,
+ShadowHandOver,8,10000,
+ShadowHandOver,9,10000,
+ShadowHandOver,10,10000,
+ShadowHandOver,11,12000,
+ShadowHandOver,12,10000,
+ShadowHandOver,13,10000,
+ShadowHandOver,14,10000,
+ShadowHandPen,3,10000,
+ShadowHandPen,4,10000,
+ShadowHandPen,5,10000,
+ShadowHandPen,6,10000,
+ShadowHandPen,7,10000,
+ShadowHandPen,8,10000,
+ShadowHandPen,9,10000,
+ShadowHandPen,10,12000,
+ShadowHandPen,11,17000,
+ShadowHandPen,12,10000,
+ShadowHandPen,13,10000,
+ShadowHandPen,14,10000,
+ShadowHandPushBlock,3,unsolved task (can solve),
+ShadowHandPushBlock,4,unsolved task (can solve),
+ShadowHandPushBlock,5,10000,
+ShadowHandPushBlock,6,10000,
+ShadowHandPushBlock,7,10000,
+ShadowHandPushBlock,8,10000,
+ShadowHandPushBlock,9,10000,
+ShadowHandPushBlock,10,unsolved task (can solve),
+ShadowHandPushBlock,11,unsolved task (can solve),
+ShadowHandPushBlock,12,10000,
+ShadowHandPushBlock,13,10000,
+ShadowHandPushBlock,14,10000,
+ShadowHandScissors,3,10000,
+ShadowHandScissors,4,10000,
+ShadowHandScissors,5,10000,
+ShadowHandScissors,6,10000,
+ShadowHandScissors,7,10000,
+ShadowHandScissors,8,10000,
+ShadowHandScissors,9,10000,
+ShadowHandScissors,10,10000,
+ShadowHandScissors,11,10000,
+ShadowHandScissors,12,10000,
+ShadowHandScissors,13,10000,
+ShadowHandScissors,14,10000,
+ShadowHandSwingCup,3,15000,
+ShadowHandSwingCup,4,13000,
+ShadowHandSwingCup,5,10000,
+ShadowHandSwingCup,6,14000,
+ShadowHandSwingCup,7,12000,
+ShadowHandSwingCup,8,12000,
+ShadowHandSwingCup,9,12000,
+ShadowHandSwingCup,10,10000,
+ShadowHandSwingCup,11,10000,
+ShadowHandSwingCup,12,10000,
+ShadowHandSwingCup,13,10000,
+ShadowHandSwingCup,14,10000,
+ShadowHandSwitch,3,10000,
+ShadowHandSwitch,4,10000,
+ShadowHandSwitch,5,10000,
+ShadowHandSwitch,6,10000,
+ShadowHandSwitch,7,10000,
+ShadowHandSwitch,8,10000,
+ShadowHandSwitch,9,10000,
+ShadowHandSwitch,10,10000,
+ShadowHandSwitch,11,10000,
+ShadowHandSwitch,12,12000,
+ShadowHandSwitch,13,10000,
+ShadowHandSwitch,14,10000,
+ShadowHandTwoCatchUnderarm,3,unsolved task (hard to solve),(but one hand solved in 11000)
+ShadowHandTwoCatchUnderarm,4,unsolved task (hard to solve),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,5,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,6,10000,
+ShadowHandTwoCatchUnderarm,7,10000,
+ShadowHandTwoCatchUnderarm,8,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,9,unsolved (must >20000),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,10,unsolved task (hard to solve),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,11,unsolved (must >20000),almost sovled in some cases from 10000
+ShadowHandTwoCatchUnderarm,12,unsolved task (hard to solve),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,13,unsolved task (hard to solve),(but one hand solved in 10000)
+ShadowHandTwoCatchUnderarm,14,unsolved (must >20000),(but one hand solved in 10000)

README.md

@@ -0,0 +1,13 @@
+---
+title: Robotinder
+emoji: 🚀
+colorFrom: indigo
+colorTo: purple
+sdk: gradio
+sdk_version: 3.12.0
+app_file: app.py
+pinned: false
+duplicated_from: quantumiracle-git/robotinder
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,351 @@
+import gradio as gr
+import os
+import random
+import numpy as np
+import pandas as pd
+import gdown
+import base64
+from time import gmtime, strftime
+from csv import writer
+import json
+import zipfile
+from os import listdir
+from os.path import isfile, join, isdir
+from datasets import load_dataset
+from hfserver import HuggingFaceDatasetSaver, HuggingFaceDatasetJSONSaver
+
+ENVS = ['ShadowHand', 'ShadowHandCatchAbreast', 'ShadowHandOver', 'ShadowHandBlockStack', 'ShadowHandCatchUnderarm',
+'ShadowHandCatchOver2Underarm', 'ShadowHandBottleCap', 'ShadowHandLiftUnderarm', 'ShadowHandTwoCatchUnderarm',
+'ShadowHandDoorOpenInward', 'ShadowHandDoorOpenOutward', 'ShadowHandDoorCloseInward', 'ShadowHandDoorCloseOutward',
+'ShadowHandPushBlock', 'ShadowHandKettle', 
+'ShadowHandScissors', 'ShadowHandPen', 'ShadowHandSwingCup', 'ShadowHandGraspAndPlace', 'ShadowHandSwitch']
+
+# download data from huggingface dataset
+# dataset = load_dataset("quantumiracle-git/robotinder-data")
+# os.remove('.git/hooks/pre-push')  # https://github.com/git-lfs/git-lfs/issues/853
+LOAD_DATA_GOOGLE_DRIVE = False
+
+if LOAD_DATA_GOOGLE_DRIVE:  # download data from google drive
+    # url = 'https://drive.google.com/drive/folders/1JuNQS4R7axTezWj1x4KRAuRt_L26ApxA?usp=sharing'  # './processed/' folder in google drive
+    # url = 'https://drive.google.com/drive/folders/1o8Q9eX-J7F326zv4g2MZWlzR46uVkUF2?usp=sharing'  # './processed_zip/' folder in google drive
+    # url = 'https://drive.google.com/drive/folders/1ZWgpPiZwnWfwlwta8Tu-Jtu2HsS7HAEa?usp=share_link'  # './filter_processed_zip/' folder in google drive
+    # url = 'https://drive.google.com/drive/folders/1ROkuX6rQpyK7vLqF5fL2mggKiMCdKSuY?usp=share_link'  # './split_processed_zip/' folder in google drive
+
+    # output = './'
+    # id = url.split('/')[-1]
+    # os.system(f"gdown --id {id} -O {output} --folder --no-cookies --remaining-ok")
+    # # VIDEO_PATH = 'processed_zip'
+    # # VIDEO_PATH = 'filter_processed_zip'
+    # VIDEO_PATH = 'split_processed_zip'
+
+    # # unzip the zip files to the same location and delete zip files
+    # path_to_zip_file = VIDEO_PATH
+    # zip_files = [join(path_to_zip_file, f) for f in listdir(path_to_zip_file)]
+    # for f in zip_files:
+    #     if f.endswith(".zip"):
+    #         directory_to_extract_to = path_to_zip_file # extracted file itself contains a folder
+    #         print(f'extract data {f} to {directory_to_extract_to}')
+    #         with zipfile.ZipFile(f, 'r') as zip_ref:
+    #             zip_ref.extractall(directory_to_extract_to)
+    #         os.remove(f)
+
+    ### multiple urls to handle the retrieve error
+    # urls = [
+    #     'https://drive.google.com/drive/folders/1BbQe4XtcsalsvwGVLW9jWCkr-ln5pvyf?usp=share_link',  # './filter_processed_zip/1' folder in google drive
+    #     'https://drive.google.com/drive/folders/1saUTUuObPhMJFguc2J_O0K5woCJjYHci?usp=share_link',  # './filter_processed_zip/2' folder in google drive
+    #     'https://drive.google.com/drive/folders/1Kh9_E28-RH8g8EP1V3DhGI7KRs9LB7YJ?usp=share_link',  # './filter_processed_zip/3' folder in google drive
+    #     'https://drive.google.com/drive/folders/1oE75Dz6hxtaSpNhjD22PmQfgQ-PjnEc0?usp=share_link',  # './filter_processed_zip/4' folder in google drive
+    #     'https://drive.google.com/drive/folders/1XSPEKFqNHpXdLho-bnkT6FZZXssW8JkC?usp=share_link',  # './filter_processed_zip/5' folder in google drive
+    #     'https://drive.google.com/drive/folders/1XwjAHqR7kF1uSyZZIydQMoETfdvi0aPD?usp=share_link',
+    #     'https://drive.google.com/drive/folders/1TceozOWhLsbqP-w-RkforjAVo1M2zsRP?usp=share_link',
+    #     'https://drive.google.com/drive/folders/1zAP9eDSW5Eh_isACuZJadXcFaJNqEM9u?usp=share_link',
+    #     'https://drive.google.com/drive/folders/1oK8fyF9A3Pv5JubvrQMjTE9n66vYlyZN?usp=share_link',
+    #     'https://drive.google.com/drive/folders/1cezGNjlM0ONMM6C0N_PbZVCGsTyVSR0w?usp=share_link',
+    # ]
+
+    urls = [
+        'https://drive.google.com/drive/folders/1SF5jQ7HakO3lFXBon57VP83-AwfnrM3F?usp=share_link',  # './split_processed_zip/1' folder in google drive
+        'https://drive.google.com/drive/folders/13WuS6ow6sm7ws7A5xzCEhR-2XX_YiIu5?usp=share_link',  # './split_processed_zip/2' folder in google drive
+        'https://drive.google.com/drive/folders/1GWLffJDOyLkubF2C03UFcB7iFpzy1aDy?usp=share_link',  # './split_processed_zip/3' folder in google drive
+        'https://drive.google.com/drive/folders/1UKAntA7WliD84AUhRN224PkW4vt9agZW?usp=share_link',  # './split_processed_zip/4' folder in google drive
+        'https://drive.google.com/drive/folders/11cCQw3qb1vJbviVPfBnOVWVzD_VzHdWs?usp=share_link',  # './split_processed_zip/5' folder in google drive
+        'https://drive.google.com/drive/folders/1Wvy604wCxEdXAwE7r3sE0L0ieXvM__u8?usp=share_link',
+        'https://drive.google.com/drive/folders/1BTv_pMTNGm7m3hD65IgBrX880v-rLIaf?usp=share_link',
+        'https://drive.google.com/drive/folders/12x7F11ln2VQkqi8-Mu3kng74eLgifM0N?usp=share_link',
+        'https://drive.google.com/drive/folders/1OWkOul2CCrqynqpt44Fu1CBxzNNfOFE2?usp=share_link',
+        'https://drive.google.com/drive/folders/1ukwsfrbSEqCBNmRSuAYvYBHijWCQh2OU?usp=share_link',
+        'https://drive.google.com/drive/folders/1EO7zumR6sVfsWQWCS6zfNs5WuO2Se6WX?usp=share_link',
+        'https://drive.google.com/drive/folders/1aw0iBWvvZiSKng0ejRK8xbNoHLVUFCFu?usp=share_link',
+        'https://drive.google.com/drive/folders/1szIcxlVyT5WJtzpqYWYlue0n82A6-xtk?usp=share_link',
+    ]
+
+    output = './'
+    # VIDEO_PATH = 'processed_zip'
+    # VIDEO_PATH = 'filter_processed_zip'
+    VIDEO_PATH = 'split_processed_zip'
+    for i, url in enumerate(urls):
+        id = url.split('/')[-1]
+        os.system(f"gdown --id {id} -O {output} --folder --no-cookies --remaining-ok")
+
+        # unzip the zip files to the same location and delete zip files
+        path_to_zip_file = str(i+1)
+        zip_files = [join(path_to_zip_file, f) for f in listdir(path_to_zip_file)]
+        for f in zip_files:
+            if f.endswith(".zip"):
+                directory_to_extract_to = VIDEO_PATH # extracted file itself contains a folder
+                print(f'extract data {f} to {directory_to_extract_to}')
+                with zipfile.ZipFile(f, 'r') as zip_ref:
+                    zip_ref.extractall(directory_to_extract_to)
+                os.remove(f)
+
+else:
+    VIDEO_PATH = 'processed-data'
+    path_to_zip_file = VIDEO_PATH
+    zip_files = [join(path_to_zip_file, f) for f in listdir(path_to_zip_file)]
+    for f in zip_files:
+        if f.endswith(".zip"):
+            directory_to_extract_to = path_to_zip_file # extracted file itself contains a folder
+            print(f'extract data {f} to {directory_to_extract_to}')
+            with zipfile.ZipFile(f, 'r') as zip_ref:
+                zip_ref.extractall(directory_to_extract_to)
+            os.remove(f)
+            
+# for test only
+# else:  # local data
+#     VIDEO_PATH = 'robotinder-data'
+
+VIDEO_INFO = os.path.join(VIDEO_PATH, 'video_info.json')
+
+def inference(video_path):
+    # for displaying mp4 with autoplay on Gradio
+    with open(video_path, "rb") as f:
+        data = f.read()
+        b64 = base64.b64encode(data).decode()
+    html = (
+            f"""
+            <video controls autoplay muted loop>
+            <source src="data:video/mp4;base64,{b64}" type="video/mp4">
+            </video> 
+            """
+    )
+    return html
+
+def video_identity(video):
+    return video
+
+def nan():
+    return None
+
+FORMAT = ['mp4', 'gif'][0]
+
+def get_huggingface_dataset():
+    try:
+        import huggingface_hub
+    except (ImportError, ModuleNotFoundError):
+        raise ImportError(
+            "Package `huggingface_hub` not found is needed "
+            "for HuggingFaceDatasetSaver. Try 'pip install huggingface_hub'."
+        )
+    HF_TOKEN = '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'  # my HF token
+    DATASET_NAME = 'crowdsourced-robotinder-demo'
+    FLAGGING_DIR = 'flag/'
+    path_to_dataset_repo = huggingface_hub.create_repo(
+        repo_id=DATASET_NAME,
+        token=HF_TOKEN,
+        private=False,
+        repo_type="dataset",
+        exist_ok=True,
+    )    
+    dataset_dir = os.path.join(DATASET_NAME, FLAGGING_DIR)
+    repo = huggingface_hub.Repository(
+        local_dir=dataset_dir,
+        clone_from=path_to_dataset_repo,
+        use_auth_token=HF_TOKEN,
+    )
+    repo.git_pull(lfs=True)
+    log_file = os.path.join(dataset_dir, "flag_data.csv")
+    return repo, log_file
+
+def update(user_choice, left, right, choose_env, data_folder=VIDEO_PATH, flag_to_huggingface=False):
+    global last_left_video_path 
+    global last_right_video_path 
+    global last_infer_left_video_path
+    global last_infer_right_video_path
+    
+    if flag_to_huggingface: # log
+        env_name = str(last_left_video_path).split('/')[1]  # 'robotinder-data/ENV_NAME/'
+        current_time = strftime("%Y-%m-%d-%H-%M-%S", gmtime())
+        info = [env_name, user_choice, last_left_video_path, last_right_video_path, current_time]
+        print(info)
+        repo, log_file = get_huggingface_dataset()
+        with open(log_file, 'a') as file: # incremental change of the file
+            writer_object = writer(file)
+            writer_object.writerow(info)
+            file.close()
+        if int(current_time.split('-')[-2]) % 5 == 0:  # push only on certain minutes
+            try:
+                repo.push_to_hub(commit_message=f"Flagged sample at {current_time}")
+            except:
+                repo.git_pull(lfs=True)  # sync with remote first
+                repo.push_to_hub(commit_message=f"Flagged sample at {current_time}")
+    if choose_env == 'Random' or choose_env == '': # random or no selection
+        envs = get_env_names()   
+        env_name = envs[random.randint(0, len(envs)-1)]
+    else:
+        env_name = choose_env
+    # choose video
+    left, right = randomly_select_videos(env_name)
+
+    last_left_video_path = left
+    last_right_video_path = right
+    last_infer_left_video_path = inference(left)
+    last_infer_right_video_path = inference(right)
+    
+    return last_infer_left_video_path, last_infer_right_video_path, env_name
+
+def replay(left, right):  
+    return left, right
+
+def parse_envs(folder=VIDEO_PATH, filter=True, MAX_ITER=20000, DEFAULT_ITER=20000):
+    """
+    return a dict of env_name: video_paths
+    """
+    files = {}
+    if filter:
+        df = pd.read_csv('Bidexhands_Video.csv')
+        # print(df)
+    for env_name in os.listdir(folder):
+        env_path = os.path.join(folder, env_name)
+        if os.path.isdir(env_path):
+            videos = os.listdir(env_path)
+            video_files = []
+            for video in videos:  # video name rule: EnvName_Alg_Seed_Timestamp_Checkpoint_video-episode-EpisodeID
+                if video.endswith(f'.{FORMAT}'):
+                    if filter:
+                        if len(video.split('_')) < 6:
+                            print(f'{video} is wrongly named.')
+                        seed = video.split('_')[2]
+                        checkpoint = video.split('_')[4]
+                        try:
+                            succeed_iteration = df.loc[(df['seed'] == int(seed)) & (df['env_name'] == str(env_name))]['succeed_iteration'].iloc[0]
+                        except:
+                            print(f'Env {env_name} with seed {seed} not found in Bidexhands_Video.csv')
+                            
+                        if 'unsolved' in succeed_iteration:
+                            continue
+                        elif pd.isnull(succeed_iteration):
+                            min_iter = DEFAULT_ITER
+                            max_iter = MAX_ITER
+                        elif '-' in succeed_iteration:
+                            [min_iter, max_iter] = succeed_iteration.split('-')
+                        else:
+                            min_iter = succeed_iteration
+                            max_iter = MAX_ITER
+
+                        # check if the checkpoint is in the valid range
+                        valid_checkpoints = np.arange(int(min_iter), int(max_iter)+1000, 1000)
+                        if int(checkpoint) not in valid_checkpoints:
+                            continue
+                    
+                    video_path = os.path.join(folder, env_name, video)
+                    video_files.append(video_path)
+                    # print(video_path)
+
+            files[env_name] = video_files
+
+    with open(VIDEO_INFO, 'w') as fp:
+        json.dump(files, fp)
+
+    return files
+
+def get_env_names():
+    with open(VIDEO_INFO, 'r') as fp:
+        files = json.load(fp)
+    return list(files.keys())
+
+def randomly_select_videos(env_name):
+    # load the parsed video info
+    with open(VIDEO_INFO, 'r') as fp:
+        files = json.load(fp)
+    env_files = files[env_name]
+    # randomly choose two videos
+    selected_video_ids = np.random.choice(len(env_files), 2, replace=False)
+    left_video_path = env_files[selected_video_ids[0]]
+    right_video_path = env_files[selected_video_ids[1]]
+    return left_video_path, right_video_path
+
+def build_interface(iter=3, data_folder=VIDEO_PATH):
+    import sys
+    import csv
+    csv.field_size_limit(sys.maxsize)
+    
+    HF_TOKEN = os.getenv('HF_TOKEN')
+    print(HF_TOKEN)
+    HF_TOKEN = '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'  # my HF token
+    ## hf_writer = gr.HuggingFaceDatasetSaver(HF_TOKEN, "crowdsourced-robotinder-demo")  # HuggingFace logger instead of local one: https://github.com/gradio-app/gradio/blob/master/gradio/flagging.py
+    ## callback = gr.CSVLogger()
+    # hf_writer = HuggingFaceDatasetSaver(HF_TOKEN, "crowdsourced-robotinder-demo")
+    # callback = hf_writer
+
+    # parse the video folder 
+    files = parse_envs()   
+    
+    # build gradio interface
+    with gr.Blocks() as demo:
+        gr.Markdown("## Here is <span style=color:cyan>RoboTinder</span>!")
+        gr.Markdown("### Select the best robot behaviour in your choice!")
+        # some initial values
+        env_name = list(files.keys())[random.randint(0, len(files)-1)] # random pick an env 
+        with gr.Row():
+            str_env_name = gr.Markdown(f"{env_name}")
+
+        # choose video
+        left_video_path, right_video_path = randomly_select_videos(env_name)
+        
+        with gr.Row():
+            if FORMAT == 'mp4':
+                # left = gr.PlayableVideo(left_video_path, label="left_video")
+                # right = gr.PlayableVideo(right_video_path, label="right_video")
+
+                infer_left_video_path = inference(left_video_path)
+                infer_right_video_path = inference(right_video_path)
+                right = gr.HTML(infer_right_video_path, label="right_video")
+                left = gr.HTML(infer_left_video_path, label="left_video")
+            else:
+                left = gr.Image(left_video_path, shape=(1024, 768), label="left_video")
+                # right = gr.Image(right_video_path).style(height=768, width=1024)
+                right = gr.Image(right_video_path, label="right_video")
+
+        global last_left_video_path 
+        last_left_video_path = left_video_path
+        global last_right_video_path 
+        last_right_video_path = right_video_path
+
+        global last_infer_left_video_path
+        last_infer_left_video_path = infer_left_video_path
+        global last_infer_right_video_path
+        last_infer_right_video_path = infer_right_video_path
+
+        # btn1 = gr.Button("Replay")
+        user_choice = gr.Radio(["Left", "Right", "Not Sure", "Both Good", "Both Bad"], label="Which one is your favorite?")
+        choose_env = gr.Radio(["Random"]+ENVS, label="Choose the next task:")
+        btn2 = gr.Button("Next")
+
+        # This needs to be called at some point prior to the first call to callback.flag()
+        # callback.setup([user_choice, left, right], "flagged_data_points")
+        
+        # btn1.click(fn=replay, inputs=[left, right], outputs=[left, right])
+        btn2.click(fn=update, inputs=[user_choice, left, right, choose_env], outputs=[left, right, str_env_name])
+
+        # We can choose which components to flag -- in this case, we'll flag all of them
+        # btn2.click(lambda *args: callback.flag(args), [user_choice, left, right], None, preprocess=False)  # not using the gradio flagging anymore
+
+    return demo
+
+if __name__ == "__main__":
+    last_left_video_path = None
+    last_right_video_path = None
+
+    demo = build_interface()
+    # demo.launch(share=True)
+    demo.launch(share=False)

hfserver.py

@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import csv
+import datetime
+import io
+import json
+import os
+import uuid
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, List, Optional
+
+import gradio as gr
+from gradio import encryptor, utils
+from gradio.documentation import document, set_documentation_group
+
+if TYPE_CHECKING:
+    from gradio.components import IOComponent
+
+set_documentation_group("flagging")
+
+
+def _get_dataset_features_info(is_new, components):
+    """
+    Takes in a list of components and returns a dataset features info
+    Parameters:
+    is_new: boolean, whether the dataset is new or not
+    components: list of components
+    Returns:
+    infos: a dictionary of the dataset features
+    file_preview_types: dictionary mapping of gradio components to appropriate string.
+    header: list of header strings
+    """
+    infos = {"flagged": {"features": {}}}
+    # File previews for certain input and output types
+    file_preview_types = {gr.Audio: "Audio", gr.Image: "Image"}
+    headers = []
+
+    # Generate the headers and dataset_infos
+    if is_new:
+
+        for component in components:
+            headers.append(component.label)
+            infos["flagged"]["features"][component.label] = {
+                "dtype": "string",
+                "_type": "Value",
+            }
+            if isinstance(component, tuple(file_preview_types)):
+                headers.append(component.label + " file")
+                for _component, _type in file_preview_types.items():
+                    if isinstance(component, _component):
+                        infos["flagged"]["features"][component.label + " file"] = {
+                            "_type": _type
+                        }
+                        break
+
+        headers.append("flag")
+        infos["flagged"]["features"]["flag"] = {
+            "dtype": "string",
+            "_type": "Value",
+        }
+
+    return infos, file_preview_types, headers
+
+
+class FlaggingCallback(ABC):
+    """
+    An abstract class for defining the methods that any FlaggingCallback should have.
+    """
+
+    @abstractmethod
+    def setup(self, components: List[IOComponent], flagging_dir: str):
+        """
+        This method should be overridden and ensure that everything is set up correctly for flag().
+        This method gets called once at the beginning of the Interface.launch() method.
+        Parameters:
+        components: Set of components that will provide flagged data.
+        flagging_dir: A string, typically containing the path to the directory where the flagging file should be storied (provided as an argument to Interface.__init__()).
+        """
+        pass
+
+    @abstractmethod
+    def flag(
+        self,
+        flag_data: List[Any],
+        flag_option: Optional[str] = None,
+        flag_index: Optional[int] = None,
+        username: Optional[str] = None,
+    ) -> int:
+        """
+        This method should be overridden by the FlaggingCallback subclass and may contain optional additional arguments.
+        This gets called every time the <flag> button is pressed.
+        Parameters:
+        interface: The Interface object that is being used to launch the flagging interface.
+        flag_data: The data to be flagged.
+        flag_option (optional): In the case that flagging_options are provided, the flag option that is being used.
+        flag_index (optional): The index of the sample that is being flagged.
+        username (optional): The username of the user that is flagging the data, if logged in.
+        Returns:
+        (int) The total number of samples that have been flagged.
+        """
+        pass
+
+
+@document()
+class SimpleCSVLogger(FlaggingCallback):
+    """
+    A simplified implementation of the FlaggingCallback abstract class
+    provided for illustrative purposes.  Each flagged sample (both the input and output data)
+    is logged to a CSV file on the machine running the gradio app.
+    Example:
+        import gradio as gr
+        def image_classifier(inp):
+            return {'cat': 0.3, 'dog': 0.7}
+        demo = gr.Interface(fn=image_classifier, inputs="image", outputs="label",
+                            flagging_callback=SimpleCSVLogger())
+    """
+
+    def __init__(self):
+        pass
+
+    def setup(self, components: List[IOComponent], flagging_dir: str):
+        self.components = components
+        self.flagging_dir = flagging_dir
+        os.makedirs(flagging_dir, exist_ok=True)
+
+    def flag(
+        self,
+        flag_data: List[Any],
+        flag_option: Optional[str] = None,
+        flag_index: Optional[int] = None,
+        username: Optional[str] = None,
+    ) -> int:
+        flagging_dir = self.flagging_dir
+        log_filepath = os.path.join(flagging_dir, "log.csv")
+
+        csv_data = []
+        for component, sample in zip(self.components, flag_data):
+            save_dir = os.path.join(
+                flagging_dir, utils.strip_invalid_filename_characters(component.label)
+            )
+            csv_data.append(
+                component.deserialize(
+                    sample,
+                    save_dir,
+                    None,
+                )
+            )
+
+        with open(log_filepath, "a", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(utils.sanitize_list_for_csv(csv_data))
+
+        with open(log_filepath, "r") as csvfile:
+            line_count = len([None for row in csv.reader(csvfile)]) - 1
+        return line_count
+
+
+@document()
+class CSVLogger(FlaggingCallback):
+    """
+    The default implementation of the FlaggingCallback abstract class. Each flagged
+    sample (both the input and output data) is logged to a CSV file with headers on the machine running the gradio app.
+    Example:
+        import gradio as gr
+        def image_classifier(inp):
+            return {'cat': 0.3, 'dog': 0.7}
+        demo = gr.Interface(fn=image_classifier, inputs="image", outputs="label",
+                            flagging_callback=CSVLogger())
+    Guides: using_flagging
+    """
+
+    def __init__(self):
+        pass
+
+    def setup(
+        self,
+        components: List[IOComponent],
+        flagging_dir: str,
+        encryption_key: Optional[str] = None,
+    ):
+        self.components = components
+        self.flagging_dir = flagging_dir
+        self.encryption_key = encryption_key
+        os.makedirs(flagging_dir, exist_ok=True)
+
+    def flag(
+        self,
+        flag_data: List[Any],
+        flag_option: Optional[str] = None,
+        flag_index: Optional[int] = None,
+        username: Optional[str] = None,
+    ) -> int:
+        flagging_dir = self.flagging_dir
+        log_filepath = os.path.join(flagging_dir, "log.csv")
+        is_new = not os.path.exists(log_filepath)
+
+        if flag_index is None:
+            csv_data = []
+            for idx, (component, sample) in enumerate(zip(self.components, flag_data)):
+                save_dir = os.path.join(
+                    flagging_dir,
+                    utils.strip_invalid_filename_characters(
+                        component.label or f"component {idx}"
+                    ),
+                )
+                if utils.is_update(sample):
+                    csv_data.append(str(sample))
+                else:
+                    csv_data.append(
+                        component.deserialize(
+                            sample,
+                            save_dir=save_dir,
+                            encryption_key=self.encryption_key,
+                        )
+                        if sample is not None
+                        else ""
+                    )
+            csv_data.append(flag_option if flag_option is not None else "")
+            csv_data.append(username if username is not None else "")
+            csv_data.append(str(datetime.datetime.now()))
+            if is_new:
+                headers = [
+                    component.label or f"component {idx}"
+                    for idx, component in enumerate(self.components)
+                ] + [
+                    "flag",
+                    "username",
+                    "timestamp",
+                ]
+
+        def replace_flag_at_index(file_content):
+            file_content = io.StringIO(file_content)
+            content = list(csv.reader(file_content))
+            header = content[0]
+            flag_col_index = header.index("flag")
+            content[flag_index][flag_col_index] = flag_option
+            output = io.StringIO()
+            writer = csv.writer(output)
+            writer.writerows(utils.sanitize_list_for_csv(content))
+            return output.getvalue()
+
+        if self.encryption_key:
+            output = io.StringIO()
+            if not is_new:
+                with open(log_filepath, "rb", encoding="utf-8") as csvfile:
+                    encrypted_csv = csvfile.read()
+                    decrypted_csv = encryptor.decrypt(
+                        self.encryption_key, encrypted_csv
+                    )
+                    file_content = decrypted_csv.decode()
+                    if flag_index is not None:
+                        file_content = replace_flag_at_index(file_content)
+                    output.write(file_content)
+            writer = csv.writer(output)
+            if flag_index is None:
+                if is_new:
+                    writer.writerow(utils.sanitize_list_for_csv(headers))
+                writer.writerow(utils.sanitize_list_for_csv(csv_data))
+            with open(log_filepath, "wb", encoding="utf-8") as csvfile:
+                csvfile.write(
+                    encryptor.encrypt(self.encryption_key, output.getvalue().encode())
+                )
+        else:
+            if flag_index is None:
+                with open(log_filepath, "a", newline="", encoding="utf-8") as csvfile:
+                    writer = csv.writer(csvfile)
+                    if is_new:
+                        writer.writerow(utils.sanitize_list_for_csv(headers))
+                    writer.writerow(utils.sanitize_list_for_csv(csv_data))
+            else:
+                with open(log_filepath, encoding="utf-8") as csvfile:
+                    file_content = csvfile.read()
+                    file_content = replace_flag_at_index(file_content)
+                with open(
+                    log_filepath, "w", newline="", encoding="utf-8"
+                ) as csvfile:  # newline parameter needed for Windows
+                    csvfile.write(utils.sanitize_list_for_csv(file_content))
+        with open(log_filepath, "r", encoding="utf-8") as csvfile:
+            line_count = len([None for row in csv.reader(csvfile)]) - 1
+        return line_count
+
+
+@document()
+class HuggingFaceDatasetSaver(FlaggingCallback):
+    """
+    A callback that saves each flagged sample (both the input and output data)
+    to a HuggingFace dataset.
+    Example:
+        import gradio as gr
+        hf_writer = gr.HuggingFaceDatasetSaver(HF_API_TOKEN, "image-classification-mistakes")
+        def image_classifier(inp):
+            return {'cat': 0.3, 'dog': 0.7}
+        demo = gr.Interface(fn=image_classifier, inputs="image", outputs="label",
+                            allow_flagging="manual", flagging_callback=hf_writer)
+    Guides: using_flagging
+    """
+
+    def __init__(
+        self,
+        hf_token: str,
+        dataset_name: str,
+        organization: Optional[str] = None,
+        private: bool = False,
+    ):
+        """
+        Parameters:
+            hf_token: The HuggingFace token to use to create (and write the flagged sample to) the HuggingFace dataset.
+            dataset_name: The name of the dataset to save the data to, e.g. "image-classifier-1"
+            organization: The organization to save the dataset under. The hf_token must provide write access to this organization. If not provided, saved under the name of the user corresponding to the hf_token.
+            private: Whether the dataset should be private (defaults to False).
+        """
+        self.hf_token = hf_token
+        self.dataset_name = dataset_name
+        self.organization_name = organization
+        self.dataset_private = private
+
+    def setup(self, components: List[IOComponent], flagging_dir: str):
+        """
+        Params:
+        flagging_dir (str): local directory where the dataset is cloned,
+        updated, and pushed from.
+        """
+        try:
+            import huggingface_hub
+        except (ImportError, ModuleNotFoundError):
+            raise ImportError(
+                "Package `huggingface_hub` not found is needed "
+                "for HuggingFaceDatasetSaver. Try 'pip install huggingface_hub'."
+            )
+        path_to_dataset_repo = huggingface_hub.create_repo(
+            # name=self.dataset_name,
+            repo_id=self.dataset_name,
+            token=self.hf_token,
+            private=self.dataset_private,
+            repo_type="dataset",
+            exist_ok=True,
+        )
+        self.path_to_dataset_repo = path_to_dataset_repo  # e.g. "https://huggingface.co/datasets/abidlabs/test-audio-10"
+        self.components = components
+        self.flagging_dir = flagging_dir
+        self.dataset_dir = os.path.join(flagging_dir, self.dataset_name)
+        self.repo = huggingface_hub.Repository(
+            local_dir=self.dataset_dir,
+            clone_from=path_to_dataset_repo,
+            use_auth_token=self.hf_token,
+        )
+        self.repo.git_pull(lfs=True)
+
+        # Should filename be user-specified?
+        self.log_file = os.path.join(self.dataset_dir, "data.csv")
+        self.infos_file = os.path.join(self.dataset_dir, "dataset_infos.json")
+
+    def flag(
+        self,
+        flag_data: List[Any],
+        flag_option: Optional[str] = None,
+        flag_index: Optional[int] = None,
+        username: Optional[str] = None,
+    ) -> int:
+        self.repo.git_pull(lfs=True)
+
+        is_new = not os.path.exists(self.log_file)
+
+        with open(self.log_file, "a", newline="", encoding="utf-8") as csvfile:
+            writer = csv.writer(csvfile)
+
+            # File previews for certain input and output types
+            infos, file_preview_types, headers = _get_dataset_features_info(
+                is_new, self.components
+            )
+
+            # Generate the headers and dataset_infos
+            if is_new:
+                writer.writerow(utils.sanitize_list_for_csv(headers))
+
+            # Generate the row corresponding to the flagged sample
+            csv_data = []
+            for component, sample in zip(self.components, flag_data):
+                save_dir = os.path.join(
+                    self.dataset_dir,
+                    utils.strip_invalid_filename_characters(component.label),
+                )
+                # filepath = component.deserialize(sample, save_dir, None)
+                if sample is not None and str(component)!='image':
+                    filepath = component.deserialize(sample, save_dir, None)
+                else:
+                    filepath = component.deserialize(sample, None, None)  # not saving image
+                csv_data.append(filepath)
+                if isinstance(component, tuple(file_preview_types)):
+                    csv_data.append(
+                        "{}/resolve/main/{}".format(self.path_to_dataset_repo, filepath)
+                    )
+            csv_data.append(flag_option if flag_option is not None else "")
+            writer.writerow(utils.sanitize_list_for_csv(csv_data))
+
+        if is_new:
+            json.dump(infos, open(self.infos_file, "w"))
+
+        with open(self.log_file, "r", encoding="utf-8") as csvfile:
+            line_count = len([None for row in csv.reader(csvfile)]) - 1
+
+        self.repo.push_to_hub(commit_message="Flagged sample #{}".format(line_count))
+
+        return line_count
+
+
+class HuggingFaceDatasetJSONSaver(FlaggingCallback):
+    """
+    A FlaggingCallback that saves flagged data to a Hugging Face dataset in JSONL format.
+    Each data sample is saved in a different JSONL file,
+    allowing multiple users to use flagging simultaneously.
+    Saving to a single CSV would cause errors as only one user can edit at the same time.
+    """
+
+    def __init__(
+        self,
+        hf_foken: str,
+        dataset_name: str,
+        organization: Optional[str] = None,
+        private: bool = False,
+        verbose: bool = True,
+    ):
+        """
+        Params:
+        hf_token (str): The token to use to access the huggingface API.
+        dataset_name (str): The name of the dataset to save the data to, e.g.
+            "image-classifier-1"
+        organization (str): The name of the organization to which to attach
+            the datasets. If None, the dataset attaches to the user only.
+        private (bool): If the dataset does not already exist, whether it
+            should be created as a private dataset or public. Private datasets
+            may require paid huggingface.co accounts
+        verbose (bool): Whether to print out the status of the dataset
+            creation.
+        """
+        self.hf_foken = hf_foken
+        self.dataset_name = dataset_name
+        self.organization_name = organization
+        self.dataset_private = private
+        self.verbose = verbose
+
+    def setup(self, components: List[IOComponent], flagging_dir: str):
+        """
+        Params:
+        components List[Component]: list of components for flagging
+        flagging_dir (str): local directory where the dataset is cloned,
+        updated, and pushed from.
+        """
+        try:
+            import huggingface_hub
+        except (ImportError, ModuleNotFoundError):
+            raise ImportError(
+                "Package `huggingface_hub` not found is needed "
+                "for HuggingFaceDatasetJSONSaver. Try 'pip install huggingface_hub'."
+            )
+        path_to_dataset_repo = huggingface_hub.create_repo(
+            # name=self.dataset_name,  https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/hf_api.py
+            repo_id=self.dataset_name,
+            token=self.hf_foken,
+            private=self.dataset_private,
+            repo_type="dataset",
+            exist_ok=True,
+        )
+        self.path_to_dataset_repo = path_to_dataset_repo  # e.g. "https://huggingface.co/datasets/abidlabs/test-audio-10"
+        self.components = components
+        self.flagging_dir = flagging_dir
+        self.dataset_dir = os.path.join(flagging_dir, self.dataset_name)
+        self.repo = huggingface_hub.Repository(
+            local_dir=self.dataset_dir,
+            clone_from=path_to_dataset_repo,
+            use_auth_token=self.hf_foken,
+        )
+        self.repo.git_pull(lfs=True)
+
+        self.infos_file = os.path.join(self.dataset_dir, "dataset_infos.json")
+
+    def flag(
+        self,
+        flag_data: List[Any],
+        flag_option: Optional[str] = None,
+        flag_index: Optional[int] = None,
+        username: Optional[str] = None,
+    ) -> int:
+        self.repo.git_pull(lfs=True)
+
+        # Generate unique folder for the flagged sample
+        unique_name = self.get_unique_name()  # unique name for folder
+        folder_name = os.path.join(
+            self.dataset_dir, unique_name
+        )  # unique folder for specific example
+        os.makedirs(folder_name)
+
+        # Now uses the existence of `dataset_infos.json` to determine if new
+        is_new = not os.path.exists(self.infos_file)
+
+        # File previews for certain input and output types
+        infos, file_preview_types, _ = _get_dataset_features_info(
+            is_new, self.components
+        )
+
+        # Generate the row and header corresponding to the flagged sample
+        csv_data = []
+        headers = []
+
+        for component, sample in zip(self.components, flag_data):
+            headers.append(component.label)
+
+            try:
+                filepath = component.save_flagged(
+                    folder_name, component.label, sample, None
+                )
+            except Exception:
+                # Could not parse 'sample' (mostly) because it was None and `component.save_flagged`
+                # does not handle None cases.
+                # for example: Label (line 3109 of components.py raises an error if data is None)
+                filepath = None
+
+            if isinstance(component, tuple(file_preview_types)):
+                headers.append(str(component.label) + " file")
+
+                csv_data.append(
+                    "{}/resolve/main/{}/{}".format(
+                        self.path_to_dataset_repo, unique_name, filepath
+                    )
+                    if filepath is not None
+                    else None
+                )
+
+            csv_data.append(filepath)
+        headers.append("flag")
+        csv_data.append(flag_option if flag_option is not None else "")
+
+        # Creates metadata dict from row data and dumps it
+        metadata_dict = {
+            header: _csv_data for header, _csv_data in zip(headers, csv_data)
+        }
+        self.dump_json(metadata_dict, os.path.join(folder_name, "metadata.jsonl"))
+
+        if is_new:
+            json.dump(infos, open(self.infos_file, "w"))
+
+        self.repo.push_to_hub(commit_message="Flagged sample {}".format(unique_name))
+        return unique_name
+
+    def get_unique_name(self):
+        id = uuid.uuid4()
+        return str(id)
+
+    def dump_json(self, thing: dict, file_path: str) -> None:
+        with open(file_path, "w+", encoding="utf8") as f:
+            json.dump(thing, f)

processed-data/processed.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a3acffa7d1146031eb7dc266f97225fec3bc3ab36f83b06e39e632dd02a831
+size 3158508471

requirements.txt

@@ -0,0 +1,4 @@
+gradio==3.12.0
+gdown==4.6.0
+git-lfs
+pandas

robotinder-data/.gitignore

@@ -0,0 +1 @@
+rl*.mp4

robotinder-data/data_info.json

@@ -0,0 +1 @@
+{"idx": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], "env": ["ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandBlockStack", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward", "ShadowHandDoorOpenInward"], "name": ["rl-video-episode-1.gif", "rl-video-episode-9.gif", "rl-video-episode-7.gif", "rl-video-episode-2.gif", "rl-video-episode-3.gif", "rl-video-episode-6.gif", "rl-video-episode-0.gif", "rl-video-episode-4.gif", "rl-video-episode-5.gif", "rl-video-episode-8.gif", "rl-video-episode-1.gif", "rl-video-episode-9.gif", "rl-video-episode-7.gif", "rl-video-episode-2.gif", "rl-video-episode-3.gif", "rl-video-episode-6.gif", "rl-video-episode-0.gif", "rl-video-episode-4.gif", "rl-video-episode-5.gif", "rl-video-episode-8.gif"]}

robotinder-data/data_parser.py

@@ -0,0 +1,27 @@
+import json
+
+from os import listdir
+from os.path import isfile, join, isdir
+mypath = './'
+onlyfolders = [f for f in listdir(mypath) if isdir(join(mypath, f))]
+
+idx = 0
+data_info = {'idx': [], 'env': [], 'name': []}
+for folder in onlyfolders:
+    print(folder)
+    fs = [join(mypath, folder, f) for f in listdir(join(mypath, folder)) if isfile(join(mypath, folder, f))]
+    # print(fs)
+    for f in fs:
+        if f.endswith(".gif"):
+            idx += 1
+            data_info['idx'].append(idx)
+            data_info['env'].append(folder)
+            data_info['name'].append(f.split('/')[-1])
+
+with open('data_info.json', 'w') as f:
+    json.dump(data_info, f)
+    
+with open('data_info.json', 'r') as f:
+    data_info = json.load(f)
+print(data_info)
+