Diffusion policy, step 100000

README.md

@@ -0,0 +1,19 @@
+---
+library_name: lerobot
+tags: [lerobot, robotics, diffusion-policy, piper]
+datasets: [Samisaliveagain/piper-hexagonal-bolt-insertion-100-demos-v1]
+---
+# Diffusion Policy: Piper hexagonal bolt insertion
+
+LeRobot Diffusion Policy trained on [`Samisaliveagain/piper-hexagonal-bolt-insertion-100-demos-v1`](https://huggingface.co/datasets/Samisaliveagain/piper-hexagonal-bolt-insertion-100-demos-v1)
+(100 teleoperated demos, AgileX Piper, cameras `overhead` and `angled`, 7-D absolute joint-position actions).
+
+- 100k steps, batch 32 (~30 epochs), fp32, Quadro RTX 6000
+- Images resized to 240x320, no crop; ResNet-18 (ImageNet init), separate encoder per camera
+- n_obs_steps 2, horizon 64, n_action_steps 32; DDIM with 10 inference steps
+- Inference: ~1.2 GB VRAM, ~110 ms per action chunk on an RTX 6000
+
+```python
+from lerobot.policies.diffusion.modeling_diffusion import DiffusionPolicy
+policy = DiffusionPolicy.from_pretrained("RoboticsCollectiveAachen/diffusion_piper_hex_bolt_insertion")
+```

config.json

@@ -0,0 +1,98 @@
+{
+    "type": "diffusion",
+    "n_obs_steps": 2,
+    "input_features": {
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                7
+            ]
+        },
+        "observation.images.overhead": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        },
+        "observation.images.angled": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                480,
+                640
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                7
+            ]
+        }
+    },
+    "device": "cuda",
+    "dtype": null,
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": false,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "pretrained_revision": null,
+    "horizon": 64,
+    "n_action_steps": 32,
+    "normalization_mapping": {
+        "VISUAL": "MEAN_STD",
+        "STATE": "MIN_MAX",
+        "ACTION": "MIN_MAX"
+    },
+    "drop_n_last_frames": 7,
+    "vision_backbone": "resnet18",
+    "resize_shape": [
+        240,
+        320
+    ],
+    "crop_ratio": 1.0,
+    "crop_shape": null,
+    "crop_is_random": true,
+    "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
+    "use_group_norm": false,
+    "spatial_softmax_num_keypoints": 32,
+    "use_separate_rgb_encoder_per_camera": true,
+    "down_dims": [
+        512,
+        1024,
+        2048
+    ],
+    "kernel_size": 5,
+    "n_groups": 8,
+    "diffusion_step_embed_dim": 128,
+    "use_film_scale_modulation": true,
+    "gradient_checkpointing": false,
+    "noise_scheduler_type": "DDIM",
+    "num_train_timesteps": 100,
+    "beta_schedule": "squaredcos_cap_v2",
+    "beta_start": 0.0001,
+    "beta_end": 0.02,
+    "prediction_type": "epsilon",
+    "clip_sample": true,
+    "clip_sample_range": 1.0,
+    "num_inference_steps": 10,
+    "compile_model": false,
+    "compile_mode": "reduce-overhead",
+    "do_mask_loss_for_padding": false,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.95,
+        0.999
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 1e-06,
+    "scheduler_name": "cosine",
+    "scheduler_warmup_steps": 500
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd6b4f9a7caacf464e96d4a9ee2c43e3750b2722dc9d0438fca19505c27f7973
+size 1111648828

policy_postprocessor.json

@@ -0,0 +1,32 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              7
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_postprocessor_step_0_unnormalizer_processor.safetensors"
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_0_unnormalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:318335201cb946f824d23d0ec13c9988e41ac67e02a9a3afa86a7a4bcbc567a0
+size 7664

policy_preprocessor.json

@@ -0,0 +1,64 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              7
+            ]
+          },
+          "observation.images.overhead": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "observation.images.angled": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              7
+            ]
+          }
+        },
+        "norm_map": {
+          "VISUAL": "MEAN_STD",
+          "STATE": "MIN_MAX",
+          "ACTION": "MIN_MAX"
+        }
+      },
+      "state_file": "policy_preprocessor_step_3_normalizer_processor.safetensors"
+    }
+  ]
+}

policy_preprocessor_step_3_normalizer_processor.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d78ee367c41583c1a76fcb420d4bf7d1b9cc6f1b4ac82e23717bed8910f5f4c6
+size 7672

train_config.json

@@ -0,0 +1,321 @@
+{
+    "dataset": {
+        "repo_id": "Samisaliveagain/piper-hexagonal-bolt-insertion-100-demos-v1",
+        "repo_type": "dataset",
+        "root": null,
+        "episodes": null,
+        "exclude_episodes": null,
+        "image_transforms": {
+            "enable": false,
+            "backend": "dataloader",
+            "gpu_compile": true,
+            "gpu_chunk_size": 32,
+            "max_num_transforms": 3,
+            "random_order": false,
+            "tfs": {
+                "brightness": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "brightness": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "contrast": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "contrast": [
+                            0.8,
+                            1.2
+                        ]
+                    }
+                },
+                "saturation": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "saturation": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "hue": {
+                    "weight": 1.0,
+                    "type": "ColorJitter",
+                    "kwargs": {
+                        "hue": [
+                            -0.05,
+                            0.05
+                        ]
+                    }
+                },
+                "sharpness": {
+                    "weight": 1.0,
+                    "type": "SharpnessJitter",
+                    "kwargs": {
+                        "sharpness": [
+                            0.5,
+                            1.5
+                        ]
+                    }
+                },
+                "affine": {
+                    "weight": 1.0,
+                    "type": "RandomAffine",
+                    "kwargs": {
+                        "degrees": [
+                            -5.0,
+                            5.0
+                        ],
+                        "translate": [
+                            0.05,
+                            0.05
+                        ]
+                    }
+                }
+            }
+        },
+        "revision": null,
+        "use_imagenet_stats": true,
+        "video_backend": "torchcodec",
+        "return_uint8": false,
+        "depth_output_unit": "mm",
+        "streaming": false,
+        "streaming_episode_pool_size": null,
+        "streaming_sampling_strategy": "remaining",
+        "streaming_prefetch_episodes": 8,
+        "streaming_byte_budget_gb": 8.0,
+        "streaming_decode_threads": 2,
+        "streaming_decoded_queue_size": 8,
+        "streaming_native_http_connections": null,
+        "streaming_native_http_subranges": 1,
+        "video_decoder_cache_size": null,
+        "eval_split": 0.0
+    },
+    "env": null,
+    "policy": {
+        "type": "diffusion",
+        "n_obs_steps": 2,
+        "input_features": {
+            "observation.state": {
+                "type": "STATE",
+                "shape": [
+                    7
+                ]
+            },
+            "observation.images.overhead": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            },
+            "observation.images.angled": {
+                "type": "VISUAL",
+                "shape": [
+                    3,
+                    480,
+                    640
+                ]
+            }
+        },
+        "output_features": {
+            "action": {
+                "type": "ACTION",
+                "shape": [
+                    7
+                ]
+            }
+        },
+        "device": "cuda",
+        "dtype": null,
+        "use_amp": false,
+        "use_peft": false,
+        "push_to_hub": false,
+        "repo_id": null,
+        "private": null,
+        "tags": null,
+        "license": null,
+        "pretrained_path": null,
+        "pretrained_revision": null,
+        "horizon": 64,
+        "n_action_steps": 32,
+        "normalization_mapping": {
+            "VISUAL": "MEAN_STD",
+            "STATE": "MIN_MAX",
+            "ACTION": "MIN_MAX"
+        },
+        "drop_n_last_frames": 7,
+        "vision_backbone": "resnet18",
+        "resize_shape": [
+            240,
+            320
+        ],
+        "crop_ratio": 1.0,
+        "crop_shape": null,
+        "crop_is_random": true,
+        "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1",
+        "use_group_norm": false,
+        "spatial_softmax_num_keypoints": 32,
+        "use_separate_rgb_encoder_per_camera": true,
+        "down_dims": [
+            512,
+            1024,
+            2048
+        ],
+        "kernel_size": 5,
+        "n_groups": 8,
+        "diffusion_step_embed_dim": 128,
+        "use_film_scale_modulation": true,
+        "gradient_checkpointing": false,
+        "noise_scheduler_type": "DDIM",
+        "num_train_timesteps": 100,
+        "beta_schedule": "squaredcos_cap_v2",
+        "beta_start": 0.0001,
+        "beta_end": 0.02,
+        "prediction_type": "epsilon",
+        "clip_sample": true,
+        "clip_sample_range": 1.0,
+        "num_inference_steps": 10,
+        "compile_model": false,
+        "compile_mode": "reduce-overhead",
+        "do_mask_loss_for_padding": false,
+        "optimizer_lr": 0.0001,
+        "optimizer_betas": [
+            0.95,
+            0.999
+        ],
+        "optimizer_eps": 1e-08,
+        "optimizer_weight_decay": 1e-06,
+        "scheduler_name": "cosine",
+        "scheduler_warmup_steps": 500
+    },
+    "reward_model": null,
+    "output_dir": "outputs/train/diffusion_piper_bolt",
+    "job_name": "diffusion_piper_bolt",
+    "resume": false,
+    "seed": 1000,
+    "cudnn_deterministic": false,
+    "num_workers": 12,
+    "batch_size": 32,
+    "prefetch_factor": 4,
+    "persistent_workers": true,
+    "dataloader_multiprocessing_context": "spawn",
+    "steps": 100000,
+    "env_eval_freq": 20000,
+    "log_freq": 200,
+    "eval_steps": 0,
+    "max_eval_samples": 0,
+    "tolerance_s": 0.0001,
+    "save_checkpoint": true,
+    "save_freq": 10000,
+    "checkpoint_format": "safetensors",
+    "use_policy_training_preset": true,
+    "optimizer": {
+        "type": "adam",
+        "lr": 0.0001,
+        "weight_decay": 1e-06,
+        "grad_clip_norm": 10.0,
+        "betas": [
+            0.95,
+            0.999
+        ],
+        "eps": 1e-08
+    },
+    "scheduler": {
+        "type": "diffuser",
+        "num_warmup_steps": 500,
+        "name": "cosine"
+    },
+    "parallelism": {
+        "dp_replicate": 1,
+        "dp_shard": 1,
+        "context_parallel": {
+            "ring_degree": 1,
+            "ulysses_degree": 1
+        },
+        "cfg_parallel": 1
+    },
+    "accelerator": {
+        "mixed_precision": "no",
+        "gradient_accumulation": {
+            "steps": 1
+        },
+        "grad_scaler": {
+            "init_scale": 65536.0,
+            "growth_factor": 2.0,
+            "backoff_factor": 0.5,
+            "growth_interval": 2000
+        },
+        "fsdp": {
+            "reshard_after_forward": true,
+            "wrap_modules": null,
+            "min_num_params": null,
+            "cpu_offload": false,
+            "ignored_modules": null
+        },
+        "ddp": {
+            "find_unused_parameters": true,
+            "gradient_as_bucket_view": false,
+            "static_graph": false
+        },
+        "compile": {
+            "enabled": false,
+            "backend": "inductor",
+            "mode": null,
+            "regional": true
+        },
+        "activation_checkpointing": {
+            "mode": "none"
+        }
+    },
+    "eval": {
+        "n_episodes": 50,
+        "batch_size": 11,
+        "use_async_envs": true,
+        "recording": false,
+        "recording_repo_id": null,
+        "recording_private": false
+    },
+    "ema": {
+        "enable": false,
+        "decay": null,
+        "update_after_step": 0,
+        "inv_gamma": 1.0,
+        "power": 0.75,
+        "min_decay": 0.0,
+        "max_decay": 0.9999,
+        "use_for_eval": true
+    },
+    "wandb": {
+        "enable": false,
+        "disable_artifact": false,
+        "project": "lerobot",
+        "entity": null,
+        "notes": null,
+        "run_id": null,
+        "resume": null,
+        "mode": null,
+        "console": "wrap",
+        "console_multipart": false,
+        "console_chunk_max_seconds": 0,
+        "add_tags": true
+    },
+    "peft": null,
+    "job": {
+        "target": null,
+        "image": "huggingface/lerobot-gpu:latest",
+        "timeout": "2d",
+        "detach": false,
+        "tags": []
+    },
+    "save_checkpoint_to_hub": false,
+    "sample_weighting": null,
+    "rename_map": {},
+    "checkpoint_path": null
+}