Initial commit.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+SmolVLA_Eng.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,85 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+datasets:
+- masato-ka/so100_grasp_lego
+tags:
+- robotics
+- act-policy
+- lerobot
+pipeline_tag: robotics
+---
+# Model Card for smolvla_block_instruction
+
+SmolVLA trained for the block handling with text instructions.
+
+
+
+<video src="SmolVLA_Eng.mp4" controls="true" width="640"></video>
+
+## How to Get Started with the Model
+
+See the [Lerobot library](https://github.com/huggingface/lerobot)
+
+We strong recommend the environment needs to be the same as the video. I use the camera of Macbook Air M2, Also The model was inference by Macbook Air M2 16GB.
+You can run this model with below command. Instruction set to control.single_task property.
+
+```bash
+python erobot/scripts/control_robot.py
+--robot.type=so100
+--control.type=record
+--control.fps=30
+--control.single_task="Transfer the blue block onto the yellow plate."
+--control.repo_id=<YOUR EVAL DATASET>
+--control.warmup_time_s=5
+--control.episode_time_s=60
+--control.reset_time_s=10
+--control.num_episodes=1
+--control.push_to_hub=false
+--control.policy.path=masato-ka/smolvla_block_instruct
+--control.display_data=true
+--control.policy.device=mps
+
+```
+
+This model trained with below instruction.
+
+```aiignore
+  - Transfer the blue block onto the yellow plate.
+  - Position the blue block a top the yellow plate.
+  - Set the blue block down on the yellow plate.
+  - Place blue block on yellow plate.
+  - Blue block goes on the yellow plate!
+  - Put the blue one on the yellow thing.
+  - Yellow plate for the blue block!
+  - Completely remove the blue block from the yellow plate.
+  - The blue block must be taken away from the yellow plate.
+  - Dislodge the blue block from the yellow plate entirely."
+  - Get that blue block off the yellow plate!
+  - Take the blue thing away from the yellow one.
+  - Blue's gotta go from the yellow plate!
+  - Remove blue block from yellow plate.
+```
+
+
+
+## Training Details
+
+Trained with [LeRobot@b536f47](https://github.com/huggingface/lerobot/tree/b536f47e3ff8c3b340fc5efa52f0ece0a7212a57).
+
+The model was trained using [LeRobot's training script](https://github.com/huggingface/lerobot/blob/b536f47e3ff8c3b340fc5efa52f0ece0a7212a57/lerobot/scripts/train.py) and with the [masato-ka/so100_nlact_block_instruct_v3](https://huggingface.co/datasets/masato-ka/so100_nlact_block_instruct_v3) dataset, using this command:
+
+```bash
+!python lerobot/scripts/train.py \
+  --dataset.repo_id=masato-ka/so100_nlact_block_instruct_v3 \
+  --policy.path=lerobot/smolvla_base \
+  --batch_size=8 \
+  --output_dir=outputs/train/smolvla \
+  --job_name=smolvla_exp03 \
+  --policy.device=cuda \
+  --steps=40000\
+  --save_freq=20000 \
+  --wandb.enable=true \
+  --wandb.project=smolvla_test
+```
+
+This took about 3h to train on an Nvida A100.

SmolVLA_Eng.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:565b6f516395efb05137e227c8d1133ebc7ab2be3af0dabbfd450f304050e530
+size 11398607

config.json

@@ -0,0 +1,81 @@
+{
+    "type": "smolvla",
+    "n_obs_steps": 1,
+    "normalization_mapping": {
+        "VISUAL": "IDENTITY",
+        "STATE": "MEAN_STD",
+        "ACTION": "MEAN_STD"
+    },
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                6
+            ]
+        },
+        "observation.images.front": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.instruction": {
+            "type": "STATE",
+            "shape": [
+                384
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                6
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "chunk_size": 50,
+    "n_action_steps": 50,
+    "max_state_dim": 32,
+    "max_action_dim": 32,
+    "resize_imgs_with_padding": [
+        512,
+        512
+    ],
+    "empty_cameras": 0,
+    "adapt_to_pi_aloha": false,
+    "use_delta_joint_actions_aloha": false,
+    "tokenizer_max_length": 48,
+    "num_steps": 10,
+    "use_cache": true,
+    "freeze_vision_encoder": true,
+    "train_expert_only": true,
+    "train_state_proj": true,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-10,
+    "optimizer_grad_clip_norm": 10.0,
+    "scheduler_warmup_steps": 1000,
+    "scheduler_decay_steps": 30000,
+    "scheduler_decay_lr": 2.5e-06,
+    "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+    "load_vlm_weights": true,
+    "add_image_special_tokens": false,
+    "attention_mode": "cross_attn",
+    "prefix_length": 0,
+    "pad_language_to": "max_length",
+    "num_expert_layers": 0,
+    "num_vlm_layers": 16,
+    "self_attn_every_n_layers": 2,
+    "expert_width_multiplier": 0.75,
+    "min_period": 0.004,
+    "max_period": 4.0
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51be8ded65145ee5070b3660d6a40b3730aa6d00639e01d54ccdad6d4246d2f5
+size 906716608

train_config.json

@@ -0,0 +1,193 @@
+{
+    "dataset": {
+        "repo_id": "masato-ka/so100_nlact_block_instruct_v3",
+        "root": null,
+        "episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec"
+    },
+    "env": null,
+    "policy": {
+        "type": "smolvla",
+        "n_obs_steps": 1,
+        "normalization_mapping": {
+            "VISUAL": "IDENTITY",
+            "STATE": "MEAN_STD",
+            "ACTION": "MEAN_STD"
+        },
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    6
+                ]
+            },
+            "observation.images.front": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.instruction": {
+                "type": "STATE",
+                "shape": [
+                    384
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    6
+                ]
+            }
+        },
+        "device": "cuda",
+        "use_amp": false,
+        "chunk_size": 50,
+        "n_action_steps": 1,
+        "max_state_dim": 32,
+        "max_action_dim": 32,
+        "resize_imgs_with_padding": [
+            512,
+            512
+        ],
+        "empty_cameras": 0,
+        "adapt_to_pi_aloha": false,
+        "use_delta_joint_actions_aloha": false,
+        "tokenizer_max_length": 48,
+        "num_steps": 10,
+        "use_cache": true,
+        "freeze_vision_encoder": true,
+        "train_expert_only": true,
+        "train_state_proj": true,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.9,
+            0.95
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-10,
+        "optimizer_grad_clip_norm": 10.0,
+        "scheduler_warmup_steps": 1000,
+        "scheduler_decay_steps": 30000,
+        "scheduler_decay_lr": 2.5e-06,
+        "vlm_model_name": "HuggingFaceTB/SmolVLM2-500M-Video-Instruct",
+        "load_vlm_weights": true,
+        "add_image_special_tokens": false,
+        "attention_mode": "cross_attn",
+        "prefix_length": 0,
+        "pad_language_to": "max_length",
+        "num_expert_layers": 0,
+        "num_vlm_layers": 16,
+        "self_attn_every_n_layers": 2,
+        "expert_width_multiplier": 0.75,
+        "min_period": 0.004,
+        "max_period": 4.0
+    },
+    "output_dir": "outputs/train/smolvla",
+    "job_name": "smolvla_exp03",
+    "resume": false,
+    "seed": 1000,
+    "num_workers": 4,
+    "batch_size": 8,
+    "steps": 100000,
+    "eval_freq": 20000,
+    "log_freq": 200,
+    "save_checkpoint": true,
+    "save_freq": 20000,
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adamw",
+        "lr": 0.0001,
+        "weight_decay": 1e-10,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.9,
+            0.95
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "cosine_decay_with_warmup",
+        "num_warmup_steps": 1000,
+        "num_decay_steps": 30000,
+        "peak_lr": 0.0001,
+        "decay_lr": 2.5e-06
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 50,
+        "use_async_envs": false
+    },
+    "wandb": {
+        "enable": true,
+        "disable_artifact": false,
+        "project": "smolvla_test",
+        "entity": null,
+        "notes": null,
+        "run_id": null,
+        "mode": null
+    }
+}