| { |
| "type": "rho", |
| "name": "rho", |
| "device": "cuda", |
| "dtype": "bfloat16", |
| "lr_scheduler": { |
| "type": "cosine_decay_with_warmup", |
| "num_warmup_steps": 2500, |
| "num_decay_steps": 40000, |
| "peak_lr": 0.0001, |
| "decay_lr": 5e-06, |
| "decay_start_step": null |
| }, |
| "optimizer": { |
| "type": "adamw", |
| "lr": 0.0001, |
| "weight_decay": 1e-10, |
| "grad_clip_norm": 10.0, |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "eps": 1e-08 |
| }, |
| "pretrained_repo_id": "microsoft/rho-base", |
| "resize_imgs_with_padding": null, |
| "embed_dim": 2048, |
| "num_heads": 16, |
| "ff_dim": 4096, |
| "norm": "adaptive", |
| "hidden_state_idx": 14, |
| "num_blocks": 12, |
| "max_seq_len": 6144, |
| "n_obs_steps": 1, |
| "chunk_size": 32, |
| "n_action_steps": 16, |
| "max_state_dim": 32, |
| "max_action_dim": 32, |
| "num_steps": 10, |
| "num_flow_samples": 8, |
| "attention_implementation": "flash_attention_2", |
| "attention_type": "cross", |
| "adaln_mode": "shared", |
| "adaln_lora_rank": 256, |
| "gqa_groups": 4, |
| "cross_attention_dim": null, |
| "vlm_layer_select": "last", |
| "shared_vlm_projection": false, |
| "kv_pos_encoding": true, |
| "freeze_vlm_backbone": false, |
| "freeze_vision_encoder": false, |
| "freeze_language_model": false, |
| "train_expert_only": false, |
| "enable_gradient_checkpointing": true, |
| "optimizer_lr": 0.0001, |
| "optimizer_lr_action_expert": null, |
| "optimizer_betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "optimizer_eps": 1e-08, |
| "optimizer_weight_decay": 1e-10, |
| "scheduler_warmup_steps": 2500, |
| "scheduler_decay_steps": 40000, |
| "scheduler_decay_lr": 5e-06, |
| "time_sampling_strategy": "beta", |
| "dropout_p": 0.02, |
| "pos_emb_method": "sinusoidal", |
| "log_hidden_state_stats": true |
| } |
|
|
