diff --git a/04_robocasa_scaling/README.md b/04_robocasa_scaling/README.md new file mode 100644 index 0000000000000000000000000000000000000000..be7bf98035d7cb17ed6d5022c8efc910a312daa5 --- /dev/null +++ b/04_robocasa_scaling/README.md @@ -0,0 +1,26 @@ +# Section 4 — Multitask control and scaling + +Paper: **Can WorldToken achieve effective multitask control and scale systematically?** + +Fifteen final scaling-grid checkpoints (N1–N3, seed 0, K=1, C_train=10) and the D300 / seed-123 BC-Transformer baseline. + +| Run | Training steps | Checkpoint file | +|---|---:|---| +| [scaling_n1_d50_seed0](scaling_n1_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` | +| [scaling_n2_d50_seed0](scaling_n2_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` | +| [scaling_n3_d50_seed0](scaling_n3_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` | +| [scaling_n1_d100_seed0](scaling_n1_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` | +| [scaling_n2_d100_seed0](scaling_n2_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` | +| [scaling_n3_d100_seed0](scaling_n3_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` | +| [bc_transformer_d300_seed123](bc_transformer_d300_seed123/) | 500,000 | `models/model_epoch_1000.pth` | +| [scaling_n1_d300_seed0](scaling_n1_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` | +| [scaling_n2_d300_seed0](scaling_n2_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` | +| [scaling_n3_d300_seed0](scaling_n3_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` | +| [scaling_n1_d1000_seed0](scaling_n1_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` | +| [scaling_n2_d1000_seed0](scaling_n2_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` | +| [scaling_n3_d1000_seed0](scaling_n3_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` | +| [scaling_n1_d2900_seed0](scaling_n1_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` | +| [scaling_n2_d2900_seed0](scaling_n2_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` | +| [scaling_n3_d2900_seed0](scaling_n3_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` | + +BC-Transformer ends at epoch 1,000, corresponding to 500,000 training steps. diff --git a/04_robocasa_scaling/bc_transformer_d300_seed123/config.json b/04_robocasa_scaling/bc_transformer_d300_seed123/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bfb734606ca23205dea7c6abeea4a9a41ec37563 --- /dev/null +++ b/04_robocasa_scaling/bc_transformer_d300_seed123/config.json @@ -0,0 +1,462 @@ +{ + "algo_name": "bc", + "experiment": { + "name": "e0_bc_xfmr_mg23_d300_seed123_official500k_cuda4", + "validate": false, + "logging": { + "terminal_output_to_txt": true, + "log_tb": true, + "log_wandb": false, + "wandb_proj_name": "robocasa_official_baselines_mg23_d300" + }, + "mse": { + "enabled": false, + "every_n_epochs": 50, + "on_save_ckpt": true, + "num_samples": 20, + "visualize": true + }, + "save": { + "enabled": true, + "every_n_seconds": null, + "every_n_epochs": 100, + "epochs": [], + "on_best_validation": false, + "on_best_rollout_return": false, + "on_best_rollout_success_rate": false + }, + "epoch_every_n_steps": 500, + "validation_epoch_every_n_steps": 10, + "env": null, + "additional_envs": null, + "render": false, + "render_video": false, + "keep_all_videos": false, + "video_skip": 5, + "rollout": { + "enabled": false, + "n": 50, + "horizon": 500, + "rate": 100, + "warmstart": 0, + "terminate_on_success": true, + "batched": false, + "num_batch_envs": 5 + }, + "env_meta_update_dict": { + "env_kwargs": { + "generative_textures": null, + "scene_split": null, + "style_ids": null, + "layout_ids": null, + "layout_and_style_ids": [ + [ + 1, + 1 + ], + [ + 2, + 2 + ], + [ + 4, + 4 + ], + [ + 6, + 9 + ], + [ + 7, + 10 + ] + ], + "randomize_cameras": false, + "obj_instance_split": "B" + } + }, + "ckpt_path": null + }, + "train": { + "data": [ + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "horizon": 700, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "horizon": 300, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "horizon": 600, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "horizon": 600, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": true, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "horizon": 600, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "horizon": 700, + "do_eval": true, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + }, + { + "path": "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "horizon": 500, + "do_eval": false, + "filter_key": "300_demos" + } + ], + "output_dir": "${EXPERIMENT_ROOT}/E0/runs", + "num_data_workers": 5, + "hdf5_cache_mode": null, + "hdf5_use_swmr": true, + "hdf5_load_next_obs": false, + "hdf5_normalize_obs": false, + "hdf5_filter_key": null, + "hdf5_validation_filter_key": null, + "seq_length": 10, + "pad_seq_length": true, + "frame_stack": 10, + "pad_frame_stack": true, + "dataset_keys": [ + "actions" + ], + "action_keys": [ + "actions" + ], + "action_config": { + "actions": { + "normalization": null + }, + "actions_abs": { + "normalization": "min_max" + }, + "action_dict/abs_pos": { + "normalization": "min_max" + }, + "action_dict/abs_rot_axis_angle": { + "normalization": "min_max", + "format": "rot_axis_angle" + }, + "action_dict/abs_rot_6d": { + "normalization": null, + "format": "rot_6d" + }, + "action_dict/rel_pos": { + "normalization": null + }, + "action_dict/rel_rot_axis_angle": { + "normalization": null, + "format": "rot_axis_angle" + }, + "action_dict/rel_rot_6d": { + "normalization": null, + "format": "rot_6d" + }, + "action_dict/gripper": { + "normalization": null + }, + "action_dict/base_mode": { + "normalization": null + } + }, + "goal_mode": null, + "cuda": true, + "batch_size": 16, + "num_epochs": 1000, + "seed": 123, + "max_grad_norm": 100.0, + "data_format": "robomimic", + "shuffled_obs_key_groups": null + }, + "algo": { + "optim_params": { + "policy": { + "optimizer_type": "adamw", + "learning_rate": { + "initial": 0.0001, + "decay_factor": 1.0, + "epoch_schedule": [ + 100 + ], + "scheduler_type": "constant_with_warmup" + }, + "regularization": { + "L2": 0.01 + } + } + }, + "loss": { + "l2_weight": 1.0, + "l1_weight": 0.0, + "cos_weight": 0.0 + }, + "actor_layer_dims": [], + "gaussian": { + "enabled": false, + "fixed_std": false, + "init_std": 0.1, + "min_std": 0.01, + "std_activation": "softplus", + "low_noise_eval": true + }, + "gmm": { + "enabled": true, + "num_modes": 5, + "min_std": 0.005, + "std_activation": "softplus", + "low_noise_eval": true + }, + "vae": { + "enabled": false, + "latent_dim": 14, + "latent_clip": null, + "kl_weight": 1.0, + "decoder": { + "is_conditioned": true, + "reconstruction_sum_across_elements": false + }, + "prior": { + "learn": false, + "is_conditioned": false, + "use_gmm": false, + "gmm_num_modes": 10, + "gmm_learn_weights": false, + "use_categorical": false, + "categorical_dim": 10, + "categorical_gumbel_softmax_hard": false, + "categorical_init_temp": 1.0, + "categorical_temp_anneal_step": 0.001, + "categorical_min_temp": 0.3 + }, + "encoder_layer_dims": [ + 300, + 400 + ], + "decoder_layer_dims": [ + 300, + 400 + ], + "prior_layer_dims": [ + 300, + 400 + ] + }, + "rnn": { + "enabled": false, + "horizon": 10, + "hidden_dim": 400, + "rnn_type": "LSTM", + "num_layers": 2, + "open_loop": false, + "kwargs": { + "bidirectional": false + } + }, + "transformer": { + "enabled": true, + "context_length": 10, + "embed_dim": 512, + "num_layers": 6, + "num_heads": 8, + "emb_dropout": 0.1, + "attn_dropout": 0.1, + "block_output_dropout": 0.1, + "sinusoidal_embedding": false, + "activation": "gelu", + "supervise_all_steps": true, + "nn_parameter_for_timesteps": true, + "pred_future_acs": true, + "causal": false + }, + "language_conditioned": true + }, + "observation": { + "modalities": { + "obs": { + "low_dim": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ], + "rgb": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "depth": [], + "scan": [] + }, + "goal": { + "low_dim": [], + "rgb": [], + "depth": [], + "scan": [] + } + }, + "encoder": { + "low_dim": { + "core_class": null, + "core_kwargs": {}, + "obs_randomizer_class": null, + "obs_randomizer_kwargs": {} + }, + "rgb": { + "core_class": "VisualCoreLanguageConditioned", + "core_kwargs": { + "feature_dimension": 64, + "backbone_class": "ResNet18ConvFiLM", + "backbone_kwargs": { + "pretrained": false, + "input_coord_conv": false + }, + "pool_class": "SpatialSoftmax", + "pool_kwargs": { + "num_kp": 32, + "learnable_temperature": false, + "temperature": 1.0, + "noise_std": 0.0 + } + }, + "obs_randomizer_class": "CropRandomizer", + "obs_randomizer_kwargs": { + "crop_height": 116, + "crop_width": 116, + "num_crops": 1, + "pos_enc": false + } + }, + "depth": { + "core_class": "VisualCore", + "core_kwargs": {}, + "obs_randomizer_class": null, + "obs_randomizer_kwargs": {} + }, + "scan": { + "core_class": "ScanCore", + "core_kwargs": {}, + "obs_randomizer_class": null, + "obs_randomizer_kwargs": {} + } + } + }, + "meta": { + "hp_base_config_file": null, + "hp_keys": [], + "hp_values": [] + } +} diff --git a/04_robocasa_scaling/bc_transformer_d300_seed123/run_info.json b/04_robocasa_scaling/bc_transformer_d300_seed123/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..f6fc11d42ad1dff693de054496dd90081ea84b9d --- /dev/null +++ b/04_robocasa_scaling/bc_transformer_d300_seed123/run_info.json @@ -0,0 +1,11 @@ +{ + "run_name": "bc_transformer_d300_seed123", + "original_run_name": "e0_bc_xfmr_mg23_d300_seed123_official500k_cuda4", + "paper_section": 4, + "parameters": { + "D": 300, + "C_train": 10, + "seed": 123 + }, + "final_training_step": 500000 +} diff --git a/04_robocasa_scaling/scaling_n1_d1000_seed0/checkpoint_step_00100000.pt b/04_robocasa_scaling/scaling_n1_d1000_seed0/checkpoint_step_00100000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a66506016c4058313d86b413657f8b2bfaeb5e2 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d1000_seed0/checkpoint_step_00100000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf58c8a3cf22a7733968c3c2d42e407fad079d4d7b747d186ae80098e742693 +size 531280901 diff --git a/04_robocasa_scaling/scaling_n1_d1000_seed0/config.json b/04_robocasa_scaling/scaling_n1_d1000_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fc065548f7090fc0787399778bfd7cd9ab05a6f9 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d1000_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "d_model": 128, + "denoising_steps": 20, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 4, + "n_layers": 1, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d1000_n1_world2/phase1grid_d1000_n1_nodyn_seed0_100k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 512, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 1, + "n_heads": 4, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0006, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 5000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg1000_seed0", + "grad_accum_steps": 2, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 100000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 512 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d1000_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0006, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 10000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 128, + "scaling_action_head_layers": 1, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 512, + "scaling_backbone_layers": 2, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 512, + "scaling_encoder_layers": 1, + "scaling_encoder_lr": 0.0006, + "scaling_expected_world_size": 2, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 512, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.0006, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n1", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.0006, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711_n5", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.0006, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d1000_n1_nodyn_seed0_100k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 512, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 1408, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 2 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 5000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n1_d1000_seed0/run_info.json b/04_robocasa_scaling/scaling_n1_d1000_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..fe0664052e79c61ab51af9839badc3f045c02bcc --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d1000_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n1_d1000_seed0", + "original_run_name": "phase1grid_d1000_n1_nodyn_seed0_100k_cuda4-5", + "paper_section": 4, + "parameters": { + "D": 1000, + "N": 1, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 100000 +} diff --git a/04_robocasa_scaling/scaling_n1_d100_seed0/checkpoint_step_00010000.pt b/04_robocasa_scaling/scaling_n1_d100_seed0/checkpoint_step_00010000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8c2ac8cfaa80f2aa8860a3a9016731704ced6e5 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d100_seed0/checkpoint_step_00010000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c14fc8c7dac158b75f7d3c2c8b3f545461f4486b6979a6c2644abdb79a7b14 +size 531280901 diff --git a/04_robocasa_scaling/scaling_n1_d100_seed0/config.json b/04_robocasa_scaling/scaling_n1_d100_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b22b822b43dedf4ccd0d79080442f6504f3eb3bf --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d100_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "d_model": 128, + "denoising_steps": 20, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 4, + "n_layers": 1, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 64, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d100_n1_world1/phase1grid_d100_n1_nodyn_seed0_10k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 512, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 1, + "n_heads": 4, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0006, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg100_seed0", + "grad_accum_steps": 3, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 10000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 512 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d100_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0006, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 1000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 128, + "scaling_action_head_layers": 1, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 512, + "scaling_backbone_layers": 2, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 512, + "scaling_encoder_layers": 1, + "scaling_encoder_lr": 0.0006, + "scaling_expected_world_size": 1, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 512, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.0006, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n1", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.0006, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711_n5", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 64, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.0006, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d100_n1_nodyn_seed0_10k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 512, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 1408, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 2 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n1_d100_seed0/run_info.json b/04_robocasa_scaling/scaling_n1_d100_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..712121314affb1ef3c97467415c64a725ff8218f --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d100_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n1_d100_seed0", + "original_run_name": "phase1grid_d100_n1_nodyn_seed0_10k_cuda6", + "paper_section": 4, + "parameters": { + "D": 100, + "N": 1, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 10000 +} diff --git a/04_robocasa_scaling/scaling_n1_d2900_seed0/checkpoint_step_00280000.pt b/04_robocasa_scaling/scaling_n1_d2900_seed0/checkpoint_step_00280000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f96cf426e4d9186c92094876fa204c00856f6162 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d2900_seed0/checkpoint_step_00280000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8e8d7446b9b077368928f26d82dbe3b41da3b878ce97f17d6ce819380eff432 +size 531280901 diff --git a/04_robocasa_scaling/scaling_n1_d2900_seed0/config.json b/04_robocasa_scaling/scaling_n1_d2900_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a4b6085e3aff9792cb1f0593f2fad1721c57e732 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d2900_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "d_model": 128, + "denoising_steps": 20, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 4, + "n_layers": 1, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d2900_n1_world4/phase1grid_d2900_n1_nodyn_seed0_280k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 512, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 1, + "n_heads": 4, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0006, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 14000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg2900_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 280000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 512 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d2900_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0006, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 28000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 128, + "scaling_action_head_layers": 1, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 512, + "scaling_backbone_layers": 2, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 512, + "scaling_encoder_layers": 1, + "scaling_encoder_lr": 0.0006, + "scaling_expected_world_size": 4, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 512, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.0006, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n1", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.0006, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711_n5", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.0006, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d2900_n1_nodyn_seed0_280k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 512, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 1408, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 2 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 14000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n1_d2900_seed0/run_info.json b/04_robocasa_scaling/scaling_n1_d2900_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..84dd8ad6bd7bc05ac92127d5e9d8e3f910ec071e --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d2900_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n1_d2900_seed0", + "original_run_name": "phase1grid_d2900_n1_nodyn_seed0_280k_cuda0-1-2-3", + "paper_section": 4, + "parameters": { + "D": 2900, + "N": 1, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 280000 +} diff --git a/04_robocasa_scaling/scaling_n1_d300_seed0/checkpoint_step_00030000.pt b/04_robocasa_scaling/scaling_n1_d300_seed0/checkpoint_step_00030000.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb2a5ca9ce083ae85f900d521e58df5c75119f52 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d300_seed0/checkpoint_step_00030000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3c340b937db28c148aeb9f4a161a2f039b86c8916fd95214b7bfaf72052fb7d +size 531280965 diff --git a/04_robocasa_scaling/scaling_n1_d300_seed0/config.json b/04_robocasa_scaling/scaling_n1_d300_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..638d44266c068623f7e810860a881f918ed626ce --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d300_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 128, + "denoising_steps": 20, + "dim_feedforward": 512, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 4, + "n_layers": 1, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d300_n1_world2/phase1grid_d300_n1_nodyn_seed0_30k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 512, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 1, + "n_heads": 4, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0006, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 1500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "300_demos", + "grad_accum_steps": 2, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 30000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 512 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d300_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0006, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 3000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 128, + "scaling_action_head_layers": 1, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 512, + "scaling_backbone_layers": 2, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 512, + "scaling_encoder_layers": 1, + "scaling_encoder_lr": 0.0006, + "scaling_expected_world_size": 2, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 512, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.0006, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n1", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.0006, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.0006, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d300_n1_nodyn_seed0_30k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 512, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 1408, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 2 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 1500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n1_d300_seed0/run_info.json b/04_robocasa_scaling/scaling_n1_d300_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..b5ade6d02d0fc7c725b0fbc8b04d6f78fa18a5e4 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d300_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n1_d300_seed0", + "original_run_name": "phase1grid_d300_n1_nodyn_seed0_30k_cuda2-4", + "paper_section": 4, + "parameters": { + "D": 300, + "N": 1, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 30000 +} diff --git a/04_robocasa_scaling/scaling_n1_d50_seed0/checkpoint_step_00005000.pt b/04_robocasa_scaling/scaling_n1_d50_seed0/checkpoint_step_00005000.pt new file mode 100644 index 0000000000000000000000000000000000000000..98ffa8026de94e237247c0aa9015d72488e232d2 --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d50_seed0/checkpoint_step_00005000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682682b296ebde324402a0edfaa296f9c13b1a912efd5ad130c48ac3a24ed9b9 +size 531280837 diff --git a/04_robocasa_scaling/scaling_n1_d50_seed0/config.json b/04_robocasa_scaling/scaling_n1_d50_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cdf143c0c5865ce7176a97f04564f29a8f816ecf --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d50_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "d_model": 128, + "denoising_steps": 20, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 4, + "n_layers": 1, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d50_n1_world1/phase1grid_d50_n1_nodyn_seed0_5k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 512, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 1, + "n_heads": 4, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0006, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 250, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg50_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 5000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 512 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n1_d50_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0006, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 500, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 128, + "scaling_action_head_layers": 1, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 512, + "scaling_backbone_layers": 2, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 512, + "scaling_encoder_layers": 1, + "scaling_encoder_lr": 0.0006, + "scaling_expected_world_size": 1, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 512, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.0006, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n1", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.0006, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711_n5", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.0006, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d50_n1_nodyn_seed0_5k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 512, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 1408, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 2 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 250, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n1_d50_seed0/run_info.json b/04_robocasa_scaling/scaling_n1_d50_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..d07da669961bd7b195fbe1a841a0463e8bb8efae --- /dev/null +++ b/04_robocasa_scaling/scaling_n1_d50_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n1_d50_seed0", + "original_run_name": "phase1grid_d50_n1_nodyn_seed0_5k_cuda6", + "paper_section": 4, + "parameters": { + "D": 50, + "N": 1, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 5000 +} diff --git a/04_robocasa_scaling/scaling_n2_d1000_seed0/checkpoint_step_00100000.pt b/04_robocasa_scaling/scaling_n2_d1000_seed0/checkpoint_step_00100000.pt new file mode 100644 index 0000000000000000000000000000000000000000..53c948988f840402afa7d9858c65a6a61d96d7f3 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d1000_seed0/checkpoint_step_00100000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5371a75babaaf64880677ac3ed6db74f29dca9f2483e58998ecc6db7b1e6a158 +size 1024355757 diff --git a/04_robocasa_scaling/scaling_n2_d1000_seed0/config.json b/04_robocasa_scaling/scaling_n2_d1000_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..361dcf52118688ee444347954c79e70f370e9880 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d1000_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d1000_n2_world2/phase1grid_d1000_n2_nodyn_seed0_100k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 5000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg1000_seed0", + "grad_accum_steps": 2, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 100000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d1000_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 10000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 2, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d1000_n2_nodyn_seed0_100k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 5000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n2_d1000_seed0/run_info.json b/04_robocasa_scaling/scaling_n2_d1000_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..bb998434fbb913240faaff32f48b1855e11caefb --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d1000_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n2_d1000_seed0", + "original_run_name": "phase1grid_d1000_n2_nodyn_seed0_100k_cuda4-5", + "paper_section": 4, + "parameters": { + "D": 1000, + "N": 2, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 100000 +} diff --git a/04_robocasa_scaling/scaling_n2_d100_seed0/checkpoint_step_00010000.pt b/04_robocasa_scaling/scaling_n2_d100_seed0/checkpoint_step_00010000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b0ccc64492d509a2a97308e588180bdd29c2f94 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d100_seed0/checkpoint_step_00010000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03b059f1c4b471497a3b3b4678cfd8b5df585be82f21eefc21b34ab5a5b65c1 +size 1024357869 diff --git a/04_robocasa_scaling/scaling_n2_d100_seed0/config.json b/04_robocasa_scaling/scaling_n2_d100_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b966b98db50d76092505d85874234757a6f14408 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d100_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_resume_n2_d100/phase1grid_d100_n2_nodyn_seed0_10k_resume_mb48_accum4.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg100_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 10000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d100_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": "${CHECKPOINT_ROOT}/checkpoint_step_00008000.pt", + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 1000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 1, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d100_n2_nodyn_seed0_10k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n2_d100_seed0/run_info.json b/04_robocasa_scaling/scaling_n2_d100_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..7f6e23ec24b8ea73c8100a0e812d6e671bb80421 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d100_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n2_d100_seed0", + "original_run_name": "phase1grid_d100_n2_nodyn_seed0_10k_cuda5", + "paper_section": 4, + "parameters": { + "D": 100, + "N": 2, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 10000 +} diff --git a/04_robocasa_scaling/scaling_n2_d2900_seed0/checkpoint_step_00280000.pt b/04_robocasa_scaling/scaling_n2_d2900_seed0/checkpoint_step_00280000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a179067c5bc2dd3f16894ce56e589c187a3b2edf --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d2900_seed0/checkpoint_step_00280000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d321278276a2f3919cc1c3b5472675ee3b7fffe32a44d2559dc1dddabae3b100 +size 1024355757 diff --git a/04_robocasa_scaling/scaling_n2_d2900_seed0/config.json b/04_robocasa_scaling/scaling_n2_d2900_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1fcea95eb1f23ae3ebdca9044635f99a2f3d0462 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d2900_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d2900_n2_world4/phase1grid_d2900_n2_nodyn_seed0_280k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 14000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg2900_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 280000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d2900_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 28000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 4, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d2900_n2_nodyn_seed0_280k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 14000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n2_d2900_seed0/run_info.json b/04_robocasa_scaling/scaling_n2_d2900_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..d3153e49ea2ad3602c99dd33bc5cfa8f2ead6385 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d2900_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n2_d2900_seed0", + "original_run_name": "phase1grid_d2900_n2_nodyn_seed0_280k_cuda1-2-4-5", + "paper_section": 4, + "parameters": { + "D": 2900, + "N": 2, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 280000 +} diff --git a/04_robocasa_scaling/scaling_n2_d300_seed0/checkpoint_step_00030000.pt b/04_robocasa_scaling/scaling_n2_d300_seed0/checkpoint_step_00030000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a82fc55227c14eb3df5e079541eb198f1e268ecc --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d300_seed0/checkpoint_step_00030000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c209e131e5438c4430d7274f764e1867023101d6a58b71bae7c233405920d8d6 +size 1024355757 diff --git a/04_robocasa_scaling/scaling_n2_d300_seed0/config.json b/04_robocasa_scaling/scaling_n2_d300_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0c16b059f747478ab381d5857590b332ceca5fd2 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d300_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d300_n2_world2/phase1grid_d300_n2_nodyn_seed0_30k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 1500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "300_demos", + "grad_accum_steps": 2, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 30000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d300_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 3000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 2, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d300_n2_nodyn_seed0_30k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 1500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n2_d300_seed0/run_info.json b/04_robocasa_scaling/scaling_n2_d300_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..62ddea55664a6e74f24b9f3a0a103f61ea113e51 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d300_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n2_d300_seed0", + "original_run_name": "phase1grid_d300_n2_nodyn_seed0_30k_cuda0-1", + "paper_section": 4, + "parameters": { + "D": 300, + "N": 2, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 30000 +} diff --git a/04_robocasa_scaling/scaling_n2_d50_seed0/checkpoint_step_00005000.pt b/04_robocasa_scaling/scaling_n2_d50_seed0/checkpoint_step_00005000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d5b20527945bda49caa2b36bd18f3b6e2a55d93 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d50_seed0/checkpoint_step_00005000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a50cdaab591898585aac9470797ee73fc08a0fec036ac0bb038d839fbaa9fe +size 1024355757 diff --git a/04_robocasa_scaling/scaling_n2_d50_seed0/config.json b/04_robocasa_scaling/scaling_n2_d50_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..37462b19ff3e9321c254b7302e81654ae1169a64 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d50_seed0/config.json @@ -0,0 +1,331 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/Phase1_5x5_wd0_newfamily/configs_d50_d100_n2_world1/phase1grid_d50_n2_nodyn_seed0_5k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 250, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg50_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 5000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/04_robocasa_scaling/scaling_n2_d50_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 500, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_batch_policy": "phase1_grid_h100_v20260711", + "scaling_batch_policy_d50_microbatch": 48, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 1, + "scaling_global_batch_size": 192, + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_phase1_grid_seed_policy": "one_training_seed_per_cell_seed0", + "scaling_phase1_topology": "phase1_5x5_seed0_v20260711", + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "phase1grid_d50_n2_nodyn_seed0_5k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/prefix04", + "scaling_suite": "phase1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 250, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_02.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n2_d50_seed0/run_info.json b/04_robocasa_scaling/scaling_n2_d50_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..0f38ea4dae1888b2e76fb6a3f2a737cb771302a8 --- /dev/null +++ b/04_robocasa_scaling/scaling_n2_d50_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n2_d50_seed0", + "original_run_name": "phase1grid_d50_n2_nodyn_seed0_5k_cuda4", + "paper_section": 4, + "parameters": { + "D": 50, + "N": 2, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 5000 +} diff --git a/04_robocasa_scaling/scaling_n3_d1000_seed0/config.json b/04_robocasa_scaling/scaling_n3_d1000_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0273a113367b6d96391bd81eb59fda2c1dcf1c2c --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d1000_seed0/config.json @@ -0,0 +1,327 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 256, + "denoising_steps": 20, + "dim_feedforward": 1024, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 8, + "n_layers": 4, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world4/e1_grid_d1000_n3_nodyn_seed0_100k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 1024, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 4, + "n_heads": 8, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0003, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 5000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg1000_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 100000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 1024 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d1000_n3_nodyn_seed0_100k_cuda0-4-5-7", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0003, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 10000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 4, + "scaling_action_head_width": 256, + "scaling_analysis_suites": [ + "e1_grid" + ], + "scaling_backbone_layers": 8, + "scaling_encoder_layers": 4, + "scaling_expected_world_size": 4, + "scaling_family_width": 1024, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 1000, + "scaling_grid_model_size": "n3", + "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.0003, + "scaling_lr_lookup_model": "n3", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.0003, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "e1_grid_d1000_n3_nodyn_seed0_100k", + "scaling_secondary_holdout_metric": "action_rmse", + "scaling_suite": "e1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 1024, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2816, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 5000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n3_d1000_seed0/run_info.json b/04_robocasa_scaling/scaling_n3_d1000_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..393a8c200752ff18fc64f542d099c2fdccdc2c8d --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d1000_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n3_d1000_seed0", + "original_run_name": "e1_grid_d1000_n3_nodyn_seed0_100k_cuda0-4-5-7", + "paper_section": 4, + "parameters": { + "D": 1000, + "N": 3, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 100000 +} diff --git a/04_robocasa_scaling/scaling_n3_d100_seed0/config.json b/04_robocasa_scaling/scaling_n3_d100_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..93e56a782d97c59b6ca187f93bf000b516582952 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d100_seed0/config.json @@ -0,0 +1,327 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 256, + "denoising_steps": 20, + "dim_feedforward": 1024, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 8, + "n_layers": 4, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world2/e1_grid_d100_n3_nodyn_seed0_10k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 1024, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 4, + "n_heads": 8, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0003, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg100_seed0", + "grad_accum_steps": 2, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 10000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 1024 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d100_n3_nodyn_seed0_10k_cuda2-3", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0003, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 1000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 4, + "scaling_action_head_width": 256, + "scaling_analysis_suites": [ + "e1_grid" + ], + "scaling_backbone_layers": 8, + "scaling_encoder_layers": 4, + "scaling_expected_world_size": 2, + "scaling_family_width": 1024, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 100, + "scaling_grid_model_size": "n3", + "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.0003, + "scaling_lr_lookup_model": "n3", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.0003, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "e1_grid_d100_n3_nodyn_seed0_10k", + "scaling_secondary_holdout_metric": "action_rmse", + "scaling_suite": "e1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 1024, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2816, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n3_d100_seed0/run_info.json b/04_robocasa_scaling/scaling_n3_d100_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..a308fd86be5d04456503b018e68538e51441a08c --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d100_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n3_d100_seed0", + "original_run_name": "e1_grid_d100_n3_nodyn_seed0_10k_cuda2-3", + "paper_section": 4, + "parameters": { + "D": 100, + "N": 3, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 10000 +} diff --git a/04_robocasa_scaling/scaling_n3_d2900_seed0/config.json b/04_robocasa_scaling/scaling_n3_d2900_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cae269dac7cf6db11923dd54b478196aa7816766 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d2900_seed0/config.json @@ -0,0 +1,327 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 256, + "denoising_steps": 20, + "dim_feedforward": 1024, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 8, + "n_layers": 4, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world4/e1_grid_d2900_n3_nodyn_seed0_280k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 1024, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 4, + "n_heads": 8, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0003, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 14000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg2900_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 280000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 1024 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d2900_n3_nodyn_seed0_280k_cuda1-2-3-6", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0003, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 28000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 4, + "scaling_action_head_width": 256, + "scaling_analysis_suites": [ + "e1_grid" + ], + "scaling_backbone_layers": 8, + "scaling_encoder_layers": 4, + "scaling_expected_world_size": 4, + "scaling_family_width": 1024, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 2900, + "scaling_grid_model_size": "n3", + "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.0003, + "scaling_lr_lookup_model": "n3", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.0003, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "e1_grid_d2900_n3_nodyn_seed0_280k", + "scaling_secondary_holdout_metric": "action_rmse", + "scaling_suite": "e1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 1024, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2816, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 14000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n3_d2900_seed0/run_info.json b/04_robocasa_scaling/scaling_n3_d2900_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..801243907edc37be19612398aeaf2e7e9e8154e7 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d2900_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n3_d2900_seed0", + "original_run_name": "e1_grid_d2900_n3_nodyn_seed0_280k_cuda1-2-3-6", + "paper_section": 4, + "parameters": { + "D": 2900, + "N": 3, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 280000 +} diff --git a/04_robocasa_scaling/scaling_n3_d300_seed0/config.json b/04_robocasa_scaling/scaling_n3_d300_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..552c8aaf2de1f85ceed385502c076f331a02c414 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d300_seed0/config.json @@ -0,0 +1,328 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 256, + "denoising_steps": 20, + "dim_feedforward": 1024, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 8, + "n_layers": 4, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world1/e1_grid_d300_n3_nodyn_seed0_30k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 1024, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 4, + "n_heads": 8, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0003, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 1500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "300_demos", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 30000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 1024 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d300_n3_nodyn_seed0_30k_cuda0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0003, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 3000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 4, + "scaling_action_head_width": 256, + "scaling_analysis_suites": [ + "e1_grid" + ], + "scaling_backbone_layers": 8, + "scaling_encoder_layers": 4, + "scaling_expected_world_size": 1, + "scaling_family_width": 1024, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 300, + "scaling_grid_model_size": "n3", + "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.0003, + "scaling_lr_lookup_model": "n3", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.0003, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_model_size": "n3", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "e1_grid_d300_n3_nodyn_seed0_30k", + "scaling_secondary_holdout_metric": "action_rmse", + "scaling_suite": "e1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 1024, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2816, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 1500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n3_d300_seed0/run_info.json b/04_robocasa_scaling/scaling_n3_d300_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..f77ea3a49c6c7855e534126a5ed333d7953bdbb7 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d300_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n3_d300_seed0", + "original_run_name": "e1_grid_d300_n3_nodyn_seed0_30k_cuda0", + "paper_section": 4, + "parameters": { + "D": 300, + "N": 3, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 30000 +} diff --git a/04_robocasa_scaling/scaling_n3_d50_seed0/config.json b/04_robocasa_scaling/scaling_n3_d50_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c0b3f60c41f8b97c0f4de92dc0fca81584cde513 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d50_seed0/config.json @@ -0,0 +1,327 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 256, + "denoising_steps": 20, + "dim_feedforward": 1024, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 8, + "n_layers": 4, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E1_scaling_grid/configs_world1/e1_grid_d50_n3_nodyn_seed0_5k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 1024, + "dropout": 0.0, + "mlp_ratio": 4, + "n_fusion_layers": 4, + "n_heads": 8, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_latent_token" + }, + "encoder_lr": 0.0003, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 250, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg50_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 5000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 1024 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E1_scaling_grid/runs/e1_grid_d50_n3_nodyn_seed0_5k_cuda6", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.0003, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": null, + "save_every": 500, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 4, + "scaling_action_head_width": 256, + "scaling_analysis_suites": [ + "e1_grid" + ], + "scaling_backbone_layers": 8, + "scaling_encoder_layers": 4, + "scaling_expected_world_size": 1, + "scaling_family_width": 1024, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 50, + "scaling_grid_model_size": "n3", + "scaling_grid_protocol": "direct_5x5_seed0_first_pass_v20260711", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.0003, + "scaling_lr_lookup_model": "n3", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.0003, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260711", + "scaling_model_axis": "full_model_excluding_cnn_stem", + "scaling_model_family": "equiscale_encoder_backbone_head_v20260708", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "nodyn", + "scaling_run_name": "e1_grid_d50_n3_nodyn_seed0_5k", + "scaling_secondary_holdout_metric": "action_rmse", + "scaling_suite": "e1_grid", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 1024, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2816, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8 + }, + "type": "continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 250, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/04_robocasa_scaling/scaling_n3_d50_seed0/run_info.json b/04_robocasa_scaling/scaling_n3_d50_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..11c69f9669a68a08a0030634424fcf5e6dffc899 --- /dev/null +++ b/04_robocasa_scaling/scaling_n3_d50_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "scaling_n3_d50_seed0", + "original_run_name": "e1_grid_d50_n3_nodyn_seed0_5k_cuda6", + "paper_section": 4, + "parameters": { + "D": 50, + "N": 3, + "K": 1, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 5000 +} diff --git a/05_token_interface/README.md b/05_token_interface/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a10544badbe6ba85b3430e722d29d32286f9cd9 --- /dev/null +++ b/05_token_interface/README.md @@ -0,0 +1,20 @@ +# Section 5 — Token interface + +Paper: **Do more tokens per timestep improve performance?** + +Ten final checkpoints: N2, K∈{4,50}, seed 0, C_train=10, across all five data settings. + +| Run | Training steps | Checkpoint file | +|---|---:|---| +| [multitoken_n2_k4_d50_seed0](multitoken_n2_k4_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` | +| [rawtoken_n2_k50_d50_seed0](rawtoken_n2_k50_d50_seed0/) | 5,000 | `checkpoint_step_00005000.pt` | +| [multitoken_n2_k4_d100_seed0](multitoken_n2_k4_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` | +| [rawtoken_n2_k50_d100_seed0](rawtoken_n2_k50_d100_seed0/) | 10,000 | `checkpoint_step_00010000.pt` | +| [multitoken_n2_k4_d300_seed0](multitoken_n2_k4_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` | +| [rawtoken_n2_k50_d300_seed0](rawtoken_n2_k50_d300_seed0/) | 30,000 | `checkpoint_step_00030000.pt` | +| [multitoken_n2_k4_d1000_seed0](multitoken_n2_k4_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` | +| [rawtoken_n2_k50_d1000_seed0](rawtoken_n2_k50_d1000_seed0/) | 100,000 | `checkpoint_step_00100000.pt` | +| [multitoken_n2_k4_d2900_seed0](multitoken_n2_k4_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` | +| [rawtoken_n2_k50_d2900_seed0](rawtoken_n2_k50_d2900_seed0/) | 280,000 | `checkpoint_step_00280000.pt` | + +The K=1 controls reuse the N2 / seed-0 checkpoints in [Section 4](../04_robocasa_scaling/README.md). diff --git a/05_token_interface/multitoken_n2_k4_d1000_seed0/config.json b/05_token_interface/multitoken_n2_k4_d1000_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..16f6758dccc0062e1d5678a386e924bbb4e09d04 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d1000_seed0/config.json @@ -0,0 +1,337 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world4/multitoken_d1000_n2_k4_nodyn_seed0_100k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "latent_token_self_attn": "full", + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "retained_tokens_per_frame": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_multi_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 5000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg1000_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 100000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d1000_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 10000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "world_tokens_per_frame_ablation" + ], + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 4, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 1000, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "multitoken_d1000_n2_v1", + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "retained_world_tokens_per_frame", + "scaling_model_family": "n2_multitoken_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "multitoken_causal_nodyn", + "scaling_run_name": "multitoken_d1000_n2_k4_nodyn_seed0_100k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "world_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 4 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 5000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/multitoken_n2_k4_d1000_seed0/run_info.json b/05_token_interface/multitoken_n2_k4_d1000_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..10e06475e8aa16a36441df13a152cbf0ec585ab3 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d1000_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "multitoken_n2_k4_d1000_seed0", + "original_run_name": "multitoken_d1000_n2_k4_nodyn_seed0_100k_cuda4-5-6-7", + "paper_section": 5, + "parameters": { + "D": 1000, + "N": 2, + "K": 4, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 100000 +} diff --git a/05_token_interface/multitoken_n2_k4_d100_seed0/config.json b/05_token_interface/multitoken_n2_k4_d100_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b7f3bec6328566f5d00349c4850fe4bf7a1df73b --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d100_seed0/config.json @@ -0,0 +1,333 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world1/multitoken_d100_n2_k4_nodyn_seed0_10k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "latent_token_self_attn": "full", + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "retained_tokens_per_frame": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_multi_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg100_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 10000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d100_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 1000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "world_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 1, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 100, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "multitoken_d100_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "retained_world_tokens_per_frame", + "scaling_model_family": "n2_multitoken_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "multitoken_causal_nodyn", + "scaling_run_name": "multitoken_d100_n2_k4_nodyn_seed0_10k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "world_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 4 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/multitoken_n2_k4_d100_seed0/run_info.json b/05_token_interface/multitoken_n2_k4_d100_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..eb7ec1bcaa59cd438aa5ee5f39386795e80a524a --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d100_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "multitoken_n2_k4_d100_seed0", + "original_run_name": "multitoken_d100_n2_k4_nodyn_seed0_10k_cuda7", + "paper_section": 5, + "parameters": { + "D": 100, + "N": 2, + "K": 4, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 10000 +} diff --git a/05_token_interface/multitoken_n2_k4_d2900_seed0/config.json b/05_token_interface/multitoken_n2_k4_d2900_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3773fed04689c92c287b2d2342e3ebac06fb533c --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d2900_seed0/config.json @@ -0,0 +1,333 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world4/multitoken_d2900_n2_k4_nodyn_seed0_280k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "latent_token_self_attn": "full", + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "retained_tokens_per_frame": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_multi_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 14000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg2900_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 280000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d2900_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 28000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "world_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 4, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 2900, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "multitoken_d2900_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "retained_world_tokens_per_frame", + "scaling_model_family": "n2_multitoken_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "multitoken_causal_nodyn", + "scaling_run_name": "multitoken_d2900_n2_k4_nodyn_seed0_280k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "world_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 4 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 14000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/multitoken_n2_k4_d2900_seed0/run_info.json b/05_token_interface/multitoken_n2_k4_d2900_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..b521558818758926f206d7b53087d8d2e550fc88 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d2900_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "multitoken_n2_k4_d2900_seed0", + "original_run_name": "multitoken_d2900_n2_k4_nodyn_seed0_280k_cuda4-5-6-7", + "paper_section": 5, + "parameters": { + "D": 2900, + "N": 2, + "K": 4, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 280000 +} diff --git a/05_token_interface/multitoken_n2_k4_d300_seed0/config.json b/05_token_interface/multitoken_n2_k4_d300_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfee099d7bf3f56f28c299cfde26480779323744 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d300_seed0/config.json @@ -0,0 +1,332 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_multitoken_n2_d300/configs_world1/multitoken_d300_n2_k4_nodyn_seed0_30k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "latent_token_self_attn": "full", + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "retained_tokens_per_frame": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_multi_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 1500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "300_demos", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 30000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E_multitoken_n2_d300/runs/multitoken_d300_n2_k4_nodyn_seed0_30k_cuda5", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 3000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "world_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 1, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 300, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "multitoken_d300_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "retained_world_tokens_per_frame", + "scaling_model_family": "n2_multitoken_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "multitoken_causal_nodyn", + "scaling_run_name": "multitoken_d300_n2_k4_nodyn_seed0_30k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "world_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "retained_tokens_per_frame": 4 + }, + "type": "multi_token_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 1500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/multitoken_n2_k4_d300_seed0/run_info.json b/05_token_interface/multitoken_n2_k4_d300_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..e3e2cbe995e62c66665e92126b8ca6b3ebfe4656 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d300_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "multitoken_n2_k4_d300_seed0", + "original_run_name": "multitoken_d300_n2_k4_nodyn_seed0_30k_cuda5", + "paper_section": 5, + "parameters": { + "D": 300, + "N": 2, + "K": 4, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 30000 +} diff --git a/05_token_interface/multitoken_n2_k4_d50_seed0/config.json b/05_token_interface/multitoken_n2_k4_d50_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d08c5948993c9893bbf08a54ec7e6ce3e61a5c1c --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d50_seed0/config.json @@ -0,0 +1,333 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_multitoken_n2/configs_world1/multitoken_d50_n2_k4_nodyn_seed0_5k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "latent_token_self_attn": "full", + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "readout_depth": 1, + "readout_mlp_ratio": 2, + "readout_queries": 4, + "retained_tokens_per_frame": 4, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_multi_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 250, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg50_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 5000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/multitoken_n2_k4_d50_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 500, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "world_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 1, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 50, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "multitoken_d50_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "retained_world_tokens_per_frame", + "scaling_model_family": "n2_multitoken_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "multitoken_causal_nodyn", + "scaling_run_name": "multitoken_d50_n2_k4_nodyn_seed0_5k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "world_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 4 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 250, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/multitoken_n2_k4_d50_seed0/run_info.json b/05_token_interface/multitoken_n2_k4_d50_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..147b1ab37a48961c7ae21acb6fb55d18a9dd5256 --- /dev/null +++ b/05_token_interface/multitoken_n2_k4_d50_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "multitoken_n2_k4_d50_seed0", + "original_run_name": "multitoken_d50_n2_k4_nodyn_seed0_5k_cuda6", + "paper_section": 5, + "parameters": { + "D": 50, + "N": 2, + "K": 4, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 5000 +} diff --git a/05_token_interface/rawtoken_n2_k50_d1000_seed0/config.json b/05_token_interface/rawtoken_n2_k50_d1000_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2d0de190b5dfe37cbace695607ddd5af79cf0e43 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d1000_seed0/config.json @@ -0,0 +1,333 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_d1000_architecture/configs_world4/rawtoken_d1000_n2_k50_noreadout_nodyn_seed0_100k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "expected_tokens_per_frame": 50, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_raw_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 5000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg1000_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 100000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E_d1000_architecture/runs/rawtoken_d1000_n2_k50_noreadout_nodyn_seed0_100k_cuda0-1-2-3", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 10000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "raw_observation_tokens_per_frame_ablation" + ], + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 4, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 1000, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "rawtoken_d1000_n2_v1", + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "raw_observation_tokens_per_frame", + "scaling_model_family": "n2_raw_token_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "rawtoken_causal_nodyn", + "scaling_run_name": "rawtoken_d1000_n2_k50_noreadout_nodyn_seed0_100k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "raw_observation_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 50 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 5000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/rawtoken_n2_k50_d1000_seed0/run_info.json b/05_token_interface/rawtoken_n2_k50_d1000_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..9e3b89af59dad0ba600cf74acb976489f29eabea --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d1000_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "rawtoken_n2_k50_d1000_seed0", + "original_run_name": "rawtoken_d1000_n2_k50_noreadout_nodyn_seed0_100k_cuda0-1-2-3", + "paper_section": 5, + "parameters": { + "D": 1000, + "N": 2, + "K": 50, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 100000 +} diff --git a/05_token_interface/rawtoken_n2_k50_d100_seed0/config.json b/05_token_interface/rawtoken_n2_k50_d100_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0a14932830c46d2d8016a972057da8391a0b3590 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d100_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_rawtoken_n2/configs_world1/rawtoken_d100_n2_k50_noreadout_nodyn_seed0_10k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "expected_tokens_per_frame": 50, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_raw_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg100_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 10000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/rawtoken_n2_k50_d100_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 1000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "raw_observation_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 1, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 100, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "rawtoken_d100_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "raw_observation_tokens_per_frame", + "scaling_model_family": "n2_raw_token_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "rawtoken_causal_nodyn", + "scaling_run_name": "rawtoken_d100_n2_k50_noreadout_nodyn_seed0_10k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "raw_observation_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 50 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/rawtoken_n2_k50_d100_seed0/run_info.json b/05_token_interface/rawtoken_n2_k50_d100_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..026ce4339b1642e43101bbf27c1004113920d571 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d100_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "rawtoken_n2_k50_d100_seed0", + "original_run_name": "rawtoken_d100_n2_k50_noreadout_nodyn_seed0_10k_cuda5", + "paper_section": 5, + "parameters": { + "D": 100, + "N": 2, + "K": 50, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 10000 +} diff --git a/05_token_interface/rawtoken_n2_k50_d2900_seed0/config.json b/05_token_interface/rawtoken_n2_k50_d2900_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..06c46c63c8dd1858e95327cf726600a8d06730eb --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d2900_seed0/config.json @@ -0,0 +1,334 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_d2900_architecture/configs_world4/rawtoken_d2900_n2_k50_noreadout_nodyn_seed0_280k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "expected_tokens_per_frame": 50, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_raw_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 14000, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg2900_seed0", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 280000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E_d2900_architecture/runs/rawtoken_d2900_n2_k50_noreadout_nodyn_seed0_280k_cuda0-1-2-3", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 28000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_d_model": 192, + "scaling_action_head_layers": 2, + "scaling_action_head_lr": 0.0003, + "scaling_analysis_suites": [ + "raw_observation_tokens_per_frame_ablation" + ], + "scaling_backbone_d_model": 768, + "scaling_backbone_layers": 4, + "scaling_encoder_d_model": 768, + "scaling_encoder_layers": 2, + "scaling_encoder_lr": 0.000425, + "scaling_expected_world_size": 4, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 2900, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "rawtoken_d2900_n2_v1", + "scaling_group_b_lr_role": null, + "scaling_latent_dim": 768, + "scaling_lr_action_head": 0.0003, + "scaling_lr_base": null, + "scaling_lr_base_frozen": false, + "scaling_lr_encoder": 0.000425, + "scaling_lr_formula_size_axis": "discrete_family_lookup", + "scaling_lr_lookup_calibration": false, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_predictor": 0.000425, + "scaling_lr_reference": null, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "raw_observation_tokens_per_frame", + "scaling_model_family": "n2_raw_token_interface_v1", + "scaling_per_rank_microbatch": 48, + "scaling_predictor_lr": 0.000425, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "rawtoken_causal_nodyn", + "scaling_run_name": "rawtoken_d2900_n2_k50_noreadout_nodyn_seed0_280k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "raw_observation_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 50 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 14000, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/rawtoken_n2_k50_d2900_seed0/run_info.json b/05_token_interface/rawtoken_n2_k50_d2900_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..759c7bba4eec514fd4c9ade68959e08dbcf4eae0 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d2900_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "rawtoken_n2_k50_d2900_seed0", + "original_run_name": "rawtoken_d2900_n2_k50_noreadout_nodyn_seed0_280k_cuda0-1-2-3", + "paper_section": 5, + "parameters": { + "D": 2900, + "N": 2, + "K": 50, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 280000 +} diff --git a/05_token_interface/rawtoken_n2_k50_d300_seed0/config.json b/05_token_interface/rawtoken_n2_k50_d300_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3735368d289d023cc5f12821c10b06e97e2bbce6 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d300_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_rawtoken_n2_d300/configs_world4/rawtoken_d300_n2_k50_noreadout_nodyn_seed0_30k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "expected_tokens_per_frame": 50, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_raw_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 1500, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "300_demos", + "grad_accum_steps": 1, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 30000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${EXPERIMENT_ROOT}/E_rawtoken_n2_d300/runs/rawtoken_d300_n2_k50_noreadout_nodyn_seed0_30k_cuda4-5-6-7", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 3000, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "raw_observation_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 4, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 300, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "rawtoken_d300_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "raw_observation_tokens_per_frame", + "scaling_model_family": "n2_raw_token_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "rawtoken_causal_nodyn", + "scaling_run_name": "rawtoken_d300_n2_k50_noreadout_nodyn_seed0_30k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "raw_observation_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 50 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 1500, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/rawtoken_n2_k50_d300_seed0/run_info.json b/05_token_interface/rawtoken_n2_k50_d300_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..2ce51a2f9202c4fa3fc98a17f9b4954f4eda2860 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d300_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "rawtoken_n2_k50_d300_seed0", + "original_run_name": "rawtoken_d300_n2_k50_noreadout_nodyn_seed0_30k_cuda4-5-6-7", + "paper_section": 5, + "parameters": { + "D": 300, + "N": 2, + "K": 50, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 30000 +} diff --git a/05_token_interface/rawtoken_n2_k50_d50_seed0/config.json b/05_token_interface/rawtoken_n2_k50_d50_seed0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..29bafdc9daf44faed8e7235c3b22e37de3aa807e --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d50_seed0/config.json @@ -0,0 +1,329 @@ +{ + "action_chunk_len": 10, + "action_decoder_emb": 256, + "action_head": { + "params": { + "activation": "gelu", + "d_model": 192, + "denoising_steps": 20, + "dim_feedforward": 768, + "dropout": 0.0, + "h_adaln_bottleneck": false, + "n_heads": 6, + "n_layers": 2, + "use_h_cross_attn": false, + "use_obs_cross_attn": false + }, + "type": "diffusion_dit" + }, + "action_head_lr": 0.0003, + "action_spec": { + "dim": 12, + "discrete_dims": [ + 6, + 11 + ] + }, + "action_weight": 1.0, + "adamw_beta1": 0.9, + "adamw_beta2": 0.95, + "adamw_eps": 1e-08, + "allow_holdout_overlap": false, + "attn_backend": "auto", + "attn_impl": "eager", + "backbone_lr": null, + "backbone_type": "qwen2", + "batch_size": 48, + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 4 + ], + "config": "${EXPERIMENT_ROOT}/E_rawtoken_n2/configs_world1/rawtoken_d50_n2_k50_noreadout_nodyn_seed0_5k.yaml", + "crops_per_demo": 8, + "d_model": 2048, + "dataset": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5" + ], + "ddp_timeout_minutes": 180.0, + "denoising_steps": 20, + "device": "cuda", + "diffusion_mlp_dims": [ + 1024, + 1024, + 1024 + ], + "diffusion_residual_style": true, + "diffusion_time_dim": 16, + "dropout": 0.1, + "dry_run_batch_size": 1, + "dry_run_data": false, + "dynamics": { + "enabled": false, + "params": {}, + "type": "none" + }, + "dynamics_decoder_type": "film", + "enable_pred_next": false, + "encoder": { + "params": { + "cnn_depth": 48, + "cnn_kernel": 5, + "cnn_mults": [ + 2, + 3, + 4, + 8, + 16 + ], + "d_model": 768, + "dropout": 0.0, + "expected_tokens_per_frame": 50, + "mlp_ratio": 4, + "n_fusion_layers": 2, + "n_heads": 6, + "rope_theta": 10000.0, + "use_proprio": true + }, + "type": "attn_fusion_raw_token" + }, + "encoder_lr": 0.000425, + "eval_at_start": false, + "eval_batch_size": 64, + "eval_crops_per_demo": 8, + "eval_every": 250, + "eval_num_workers": 24, + "eval_seed": 0, + "ffn_hidden_size": 1024, + "filter_key": "wl23_mg50_seed0", + "grad_accum_steps": 4, + "grad_clip": 1.0, + "hdf5_paths": [ + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeePressButton/mg/2024-05-04-22-21-32/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeServeMug/mg/2024-05-04-22-21-50/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_coffee/CoffeeSetupMug/mg/2024-05-04-22-22-13_and_2024-05-08-05-52-13/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseDoubleDoor/mg/2024-05-04-22-22-42_and_2024-05-08-06-02-36/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/CloseSingleDoor/mg/2024-05-04-22-34-56/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_doors/OpenSingleDoor/mg/2024-05-04-22-37-39/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/CloseDrawer/mg/2024-05-09-09-32-19/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_drawer/OpenDrawer/mg/2024-05-04-22-38-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOffMicrowave/mg/2024-05-04-22-39-23/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_microwave/TurnOnMicrowave/mg/2024-05-04-22-40-00/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCabToCounter/mg/2024-07-12-04-33-29/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToCab/mg/2024-05-04-22-12-27_and_2024-05-07-07-39-33/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToMicrowave/mg/2024-05-04-22-13-21_and_2024-05-07-07-41-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToSink/mg/2024-05-04-22-14-06_and_2024-05-07-07-40-17/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPCounterToStove/mg/2024-05-04-22-14-20/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPMicrowaveToCounter/mg/2024-05-04-22-14-26_and_2024-05-07-07-41-42/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPSinkToCounter/mg/2024-05-04-22-14-34_and_2024-05-07-07-40-21/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_pnp/PnPStoveToCounter/mg/2024-05-04-22-14-40/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOffSinkFaucet/mg/2024-05-04-22-17-26/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnOnSinkFaucet/mg/2024-05-04-22-17-46/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_sink/TurnSinkSpout/mg/2024-05-09-09-31-12/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOffStove/mg/2024-05-08-09-20-45/demo_gentex_im128_randcams.hdf5", + "${DATA_ROOT}/robocasa/mg_im/v0.1/single_stage/kitchen_stove/TurnOnStove/mg/2024-05-08-09-20-31/demo_gentex_im128_randcams.hdf5" + ], + "holdout_action_mean_samples": 1, + "holdout_crops_per_demo": 8, + "holdout_filter_key": "wl23_holdout100pt_seed0", + "holdout_per_task_metrics": true, + "holdout_rmse_samplers": [ + "deterministic", + "stochastic" + ], + "holdout_sample_deterministic": null, + "holdout_sample_prefix_horizon": 4, + "holdout_sample_rmse": true, + "holdout_size": 2300, + "image_emb_dim": 512, + "init_device": "meta", + "input_norm": true, + "lang_emb_cache": "${EXPERIMENT_ROOT}/robocasa_mg23_clip_lang_emb_cache.npz", + "lang_emb_mode": "clip", + "lang_encoder_device": "auto", + "lang_obs_emb_dim": 256, + "log_every": 25, + "lora": false, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_rank": 16, + "lora_target": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lr": 0.0003, + "max_context_len": 1024, + "max_steps": 5000, + "min_lr_ratio": 0.1, + "model": { + "action_chunk_len": 10, + "latent_dim": 768 + }, + "model_dtype": "float32", + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 2, + "norm_bias_weight_decay": null, + "norm_fit_batches": 0, + "num_workers": 24, + "objective": "robocasa_lang_as_obs_image_state_diffusion_action", + "obs_spec": { + "image_channels": 3, + "image_hw": [ + 128, + 128 + ], + "image_keys": [ + "robot0_agentview_left_image", + "robot0_agentview_right_image", + "robot0_eye_in_hand_image" + ], + "lang_dim": 768, + "layout": "NHWC", + "low_dim_dims": [ + 3, + 4, + 3, + 4, + 2 + ], + "low_dim_keys": [ + "robot0_base_to_eef_pos", + "robot0_base_to_eef_quat", + "robot0_base_pos", + "robot0_base_quat", + "robot0_gripper_qpos" + ] + }, + "obs_stride": 4, + "output_dir": "${RELEASE_ROOT}/05_token_interface/rawtoken_n2_k50_d50_seed0", + "persistent_workers": true, + "precision": "bf16", + "pred_next_image_key_weights": {}, + "pred_next_image_weight": 1.0, + "pred_next_lang_weight": 0.1, + "pred_next_mode": "terminal", + "pred_next_obs_offset": 1, + "pred_next_proprio_weight": 0.15, + "pred_next_steps": 4, + "pred_warmup_steps": 0, + "predictor_lr": 0.000425, + "prefetch_factor": 4, + "prepare_lang_cache_only": false, + "proprio_emb_dim": 128, + "recon": { + "enabled": false, + "params": { + "depth": 48, + "kernel_size": 5, + "mults": [ + 2, + 3, + 4, + 4 + ] + } + }, + "recon_image_weight": 1.0, + "recon_lang_weight": 0.1, + "recon_obs": false, + "recon_proprio_weight": 1.0, + "resume": null, + "resume_final_lr": null, + "resume_lr_schedule": "checkpoint", + "robomimic_src": "${EXTERNAL_ROOT}/asset_977d654eb1/robomimic", + "save_every": 500, + "save_holdout_traces": true, + "save_latest_checkpoint": true, + "scaling_action_head_layers": 2, + "scaling_action_head_width": 192, + "scaling_analysis_suites": [ + "raw_observation_tokens_per_frame_ablation" + ], + "scaling_backbone_layers": 4, + "scaling_encoder_layers": 2, + "scaling_expected_world_size": 1, + "scaling_family_width": 768, + "scaling_global_batch_size": 192, + "scaling_grid_data_demos_per_task": 50, + "scaling_grid_model_size": "n2", + "scaling_grid_protocol": "rawtoken_d50_n2_v1", + "scaling_lr_action_head": 0.0003, + "scaling_lr_encoder": 0.000425, + "scaling_lr_lookup_model": "n2", + "scaling_lr_lookup_status": "confirmed", + "scaling_lr_param_groups": [ + "encoder", + "predictor", + "action_head" + ], + "scaling_lr_predictor": 0.000425, + "scaling_lr_rule": "discrete_n_lookup_v20260711", + "scaling_lr_table_version": "n_lookup_v20260712_n4_srcenter", + "scaling_model_axis": "raw_observation_tokens_per_frame", + "scaling_model_family": "n2_raw_token_interface_v1", + "scaling_model_size": "n2", + "scaling_per_rank_microbatch": 48, + "scaling_primary_holdout_metric": "task_macro/action_ddpm_loss", + "scaling_recipe": "rawtoken_causal_nodyn", + "scaling_run_name": "rawtoken_d50_n2_k50_noreadout_nodyn_seed0_5k", + "scaling_secondary_holdout_metric": "task_macro/action_rmse/stochastic/full10", + "scaling_suite": "raw_observation_tokens_per_frame_ablation", + "seed": 0, + "seq_len": 10, + "sequence_model": { + "backbone_type": "qwen2", + "hidden_dim": 768, + "params": { + "attn_impl": "sdpa", + "dropout": 0.1, + "ffn_hidden_size": 2048, + "frame_readout_index": -1, + "input_norm": false, + "max_context_len": 1024, + "model_dtype": "float32", + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 4, + "tokens_per_frame": 50 + }, + "type": "frame_major_continuous_transformer" + }, + "use_proprio": true, + "warmup_steps": 250, + "weight_decay": 0.0, + "z_bottleneck": { + "dim": null + }, + "eval_window_spec": "${CODE_ROOT}/experiments/common/eval_windows/c10_01.json.gz" +} diff --git a/05_token_interface/rawtoken_n2_k50_d50_seed0/run_info.json b/05_token_interface/rawtoken_n2_k50_d50_seed0/run_info.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba02d53ec06114c975258ca4868ac3cdc51ae65 --- /dev/null +++ b/05_token_interface/rawtoken_n2_k50_d50_seed0/run_info.json @@ -0,0 +1,13 @@ +{ + "run_name": "rawtoken_n2_k50_d50_seed0", + "original_run_name": "rawtoken_d50_n2_k50_noreadout_nodyn_seed0_5k_cuda4", + "paper_section": 5, + "parameters": { + "D": 50, + "N": 2, + "K": 50, + "C_train": 10, + "seed": 0 + }, + "final_training_step": 5000 +}