Publish Balance FastTD3 zero-latency teacher

zero-latency/humanoidbench-balance-simple/small-policy/fasttd3-v1/checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1938b393dbe08ff6287b80c6b1430965a0c396518e90a8665cff4f4d0d47ba2b
+size 33060235

zero-latency/humanoidbench-balance-simple/small-policy/fasttd3-v1/config.json

@@ -0,0 +1,260 @@
+{
+  "help": false,
+  "algo": "FAST_TD3",
+  "env": "latency_gymnasium",
+  "experiment": "g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1",
+  "train_dir": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints",
+  "restart_behavior": "resume",
+  "device": "gpu",
+  "seed": 1,
+  "num_policies": 1,
+  "async_rl": false,
+  "serial_mode": false,
+  "batched_sampling": false,
+  "num_batches_to_accumulate": 2,
+  "worker_num_splits": 1,
+  "policy_workers_per_policy": 1,
+  "max_policy_lag": 300,
+  "num_workers": 128,
+  "num_envs_per_worker": 1,
+  "batch_size": 128,
+  "num_batches_per_epoch": 1,
+  "num_epochs": 1,
+  "rollout": 1,
+  "fasttd3_replay_capacity": 6553600,
+  "fasttd3_replay_batch_size": 32768,
+  "fasttd3_transitions_per_update": 64,
+  "fasttd3_v_min": -250.0,
+  "fasttd3_v_max": 250.0,
+  "fasttd3_actor_action_l2": 0.0,
+  "fasttd3_compile": true,
+  "fasttd3_sonic_decoder_path": null,
+  "recurrence": 1,
+  "shuffle_minibatches": false,
+  "gamma": 0.99,
+  "reward_scale": 1.0,
+  "reward_clip": 1000000000.0,
+  "value_bootstrap": false,
+  "normalize_returns": false,
+  "exploration_loss_coeff": 0.0,
+  "value_loss_coeff": 0.5,
+  "kl_loss_coeff": 0.1,
+  "exploration_loss": "entropy",
+  "gae_lambda": 0.95,
+  "ppo_clip_ratio": 0.1,
+  "ppo_clip_value": 0.2,
+  "with_vtrace": false,
+  "vtrace_rho": 1.0,
+  "vtrace_c": 1.0,
+  "optimizer": "adam",
+  "adam_eps": 1e-06,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "max_grad_norm": 4.0,
+  "learning_rate": 0.0003,
+  "lr_schedule": "constant",
+  "lr_schedule_kl_threshold": 0.008,
+  "lr_adaptive_min": 1e-06,
+  "lr_adaptive_max": 0.01,
+  "obs_subtract_mean": 0.0,
+  "obs_scale": 1.0,
+  "normalize_input": false,
+  "normalize_input_keys": null,
+  "decorrelate_experience_max_seconds": 0,
+  "decorrelate_envs_on_one_worker": true,
+  "actor_worker_gpus": [],
+  "set_workers_cpu_affinity": true,
+  "force_envs_single_thread": false,
+  "default_niceness": 0,
+  "log_to_file": true,
+  "experiment_summaries_interval": 10,
+  "flush_summaries_interval": 30,
+  "stats_avg": 100,
+  "summaries_use_frameskip": true,
+  "heartbeat_interval": 20,
+  "heartbeat_reporting_interval": 180,
+  "train_for_env_steps": 25000000,
+  "train_for_seconds": 10000000000,
+  "save_every_sec": 60,
+  "keep_checkpoints": 5,
+  "load_checkpoint_kind": "latest",
+  "save_milestones_sec": -1,
+  "save_best_every_sec": 5,
+  "save_best_metric": "reward",
+  "save_best_after": 100000,
+  "benchmark": false,
+  "encoder_mlp_layers": [
+    64,
+    64
+  ],
+  "encoder_conv_architecture": "convnet_simple",
+  "encoder_conv_mlp_layers": [
+    512
+  ],
+  "use_rnn": false,
+  "rnn_size": 512,
+  "rnn_type": "gru",
+  "rnn_num_layers": 1,
+  "decoder_mlp_layers": [],
+  "nonlinearity": "elu",
+  "policy_initialization": "orthogonal",
+  "policy_init_gain": 1.0,
+  "actor_critic_share_weights": true,
+  "adaptive_stddev": false,
+  "continuous_tanh_scale": 0.0,
+  "initial_stddev": 1.0,
+  "use_env_info_cache": false,
+  "env_gpu_actions": false,
+  "env_gpu_observations": true,
+  "env_frameskip": 1,
+  "env_framestack": 1,
+  "pixel_format": "CHW",
+  "use_record_episode_statistics": false,
+  "with_wandb": true,
+  "wandb_user": null,
+  "wandb_project": "latency-sensitive-bench",
+  "wandb_group": "g1-sonic-balance-fasttd3",
+  "wandb_job_type": "sample_factory",
+  "wandb_tags": [
+    "humanoidbench",
+    "g1",
+    "balance",
+    "fasttd3"
+  ],
+  "with_pbt": false,
+  "pbt_mix_policies_in_one_env": true,
+  "pbt_period_env_steps": 5000000,
+  "pbt_start_mutation": 20000000,
+  "pbt_replace_fraction": 0.3,
+  "pbt_mutation_rate": 0.15,
+  "pbt_replace_reward_gap": 0.1,
+  "pbt_replace_reward_gap_absolute": 1e-06,
+  "pbt_optimize_gamma": false,
+  "pbt_target_objective": "true_objective",
+  "pbt_perturb_min": 1.1,
+  "pbt_perturb_max": 1.5,
+  "gym_task_name": "g1_sonic_balance_simple",
+  "gym_env_id": "LatencyBench/G1SonicBalanceSimple-v0",
+  "gym_make_kwargs_json": "{\"render_mode\": null, \"width\": 256, \"height\": 256, \"scene_path\": \"/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_sonic_assets/assets/sonic_g1/envs/scene_balance_simple.xml\", \"control_mode\": \"raw_body_target\", \"target_velocity_ramp_time_s\": 0.1, \"raw_learn_hands\": true}",
+  "gym_registration_imports_json": "[\"latency_bench.envs.humanoidbench_sonic\"]",
+  "gym_action_space_json": "{\"type\": \"box\", \"labels\": [\"body_0\", \"body_1\", \"body_2\", \"body_3\", \"body_4\", \"body_5\", \"body_6\", \"body_7\", \"body_8\", \"body_9\", \"body_10\", \"body_11\", \"body_12\", \"body_13\", \"body_14\", \"body_15\", \"body_16\", \"body_17\", \"body_18\", \"body_19\", \"body_20\", \"body_21\", \"body_22\", \"body_23\", \"body_24\", \"body_25\", \"body_26\", \"body_27\", \"body_28\", \"hand_0\", \"hand_1\", \"hand_2\", \"hand_3\", \"hand_4\", \"hand_5\", \"hand_6\", \"hand_7\", \"hand_8\", \"hand_9\", \"hand_10\", \"hand_11\", \"hand_12\", \"hand_13\"], \"low\": [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], \"high\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"dtype\": \"float32\"}",
+  "gym_action_labels_json": null,
+  "gym_action_values_json": null,
+  "gym_noop_action_id": null,
+  "gym_noop_action_json": "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",
+  "gym_base_prompt": "Keep the G1 humanoid balanced on the support board.",
+  "gym_state_labels_json": "",
+  "env_fps": 50.0,
+  "obs_fps": 50.0,
+  "frame_stack": 1,
+  "export_env_raw_rgb_frames": false,
+  "mode": "train",
+  "latency_type": "zero",
+  "fixed_latency_ms": null,
+  "mean_latency_ms": null,
+  "std_latency_ms": null,
+  "min_latency_ms": null,
+  "max_latency_ms": null,
+  "latency_seed": null,
+  "add_latency_info": false,
+  "max_pending_actions": null,
+  "hold_policy": "hold",
+  "ordering_policy": "issue_order_fifo",
+  "eval_episodes": 1,
+  "eval_parallel_envs": 1,
+  "eval_latency_raw_frame_values": null,
+  "eval_max_steps": 1000,
+  "eval_deterministic": true,
+  "eval_raw_reward": false,
+  "episode_metrics_path": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints/g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1/episode_metrics.jsonl",
+  "command_line": "--mode train --algo FAST_TD3 --env latency_gymnasium --experiment g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1 --train_dir /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints --restart_behavior resume --device gpu --actor_worker_gpus --seed 1 --episode_metrics_path /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints/g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1/episode_metrics.jsonl --train_for_env_steps 25000000 --num_workers 128 --num_envs_per_worker 1 --num_policies 1 --batch_size 128 --rollout 1 --recurrence 1 --num_epochs 1 --num_batches_per_epoch 1 --worker_num_splits 1 --max_policy_lag 300 --learning_rate 0.0003 --kl_loss_coeff 0.1 --gamma 0.99 --gae_lambda 0.95 --ppo_clip_ratio 0.1 --ppo_clip_value 0.2 --exploration_loss_coeff 0.0 --reward_scale 1.0 --reward_clip 1000000000.0 --value_loss_coeff 0.5 --max_grad_norm 4.0 --save_every_sec 60 --keep_checkpoints 5 --fasttd3_replay_capacity 6553600 --fasttd3_replay_batch_size 32768 --fasttd3_transitions_per_update 64 --fasttd3_v_min -250.0 --fasttd3_v_max 250.0 --async_rl False --batched_sampling False --use_rnn False --normalize_returns False --normalize_input False --adaptive_stddev False --actor_critic_share_weights True --fasttd3_compile True --encoder_mlp_layers 64 64 --latency-type zero --add-latency-info False --eval-episodes 1 --eval-parallel-envs 1 --eval-max-steps 1000 --eval-deterministic True --with_wandb True --wandb_project latency-sensitive-bench --wandb_group g1-sonic-balance-fasttd3 --wandb_job_type sample_factory --wandb_tags humanoidbench g1 balance fasttd3 --gym-task-name g1_sonic_balance_simple --gym-env-id LatencyBench/G1SonicBalanceSimple-v0 --gym-make-kwargs-json {\"render_mode\": null, \"width\": 256, \"height\": 256, \"scene_path\": \"/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_sonic_assets/assets/sonic_g1/envs/scene_balance_simple.xml\", \"control_mode\": \"raw_body_target\", \"target_velocity_ramp_time_s\": 0.1, \"raw_learn_hands\": true} --gym-registration-imports-json [\"latency_bench.envs.humanoidbench_sonic\"] --gym-action-space-json {\"type\": \"box\", \"labels\": [\"body_0\", \"body_1\", \"body_2\", \"body_3\", \"body_4\", \"body_5\", \"body_6\", \"body_7\", \"body_8\", \"body_9\", \"body_10\", \"body_11\", \"body_12\", \"body_13\", \"body_14\", \"body_15\", \"body_16\", \"body_17\", \"body_18\", \"body_19\", \"body_20\", \"body_21\", \"body_22\", \"body_23\", \"body_24\", \"body_25\", \"body_26\", \"body_27\", \"body_28\", \"hand_0\", \"hand_1\", \"hand_2\", \"hand_3\", \"hand_4\", \"hand_5\", \"hand_6\", \"hand_7\", \"hand_8\", \"hand_9\", \"hand_10\", \"hand_11\", \"hand_12\", \"hand_13\"], \"low\": [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], \"high\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"dtype\": \"float32\"} --gym-noop-action-json [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] --gym-base-prompt Keep the G1 humanoid balanced on the support board. --env-fps 50.0 --obs-fps 50.0 --frame-stack 1 --hold-policy hold --ordering-policy issue_order_fifo --use_env_info_cache False",
+  "cli_args": {
+    "algo": "FAST_TD3",
+    "env": "latency_gymnasium",
+    "experiment": "g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1",
+    "train_dir": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints",
+    "restart_behavior": "resume",
+    "device": "gpu",
+    "seed": 1,
+    "num_policies": 1,
+    "async_rl": false,
+    "batched_sampling": false,
+    "worker_num_splits": 1,
+    "max_policy_lag": 300,
+    "num_workers": 128,
+    "num_envs_per_worker": 1,
+    "batch_size": 128,
+    "num_batches_per_epoch": 1,
+    "num_epochs": 1,
+    "rollout": 1,
+    "fasttd3_replay_capacity": 6553600,
+    "fasttd3_replay_batch_size": 32768,
+    "fasttd3_transitions_per_update": 64,
+    "fasttd3_v_min": -250.0,
+    "fasttd3_v_max": 250.0,
+    "fasttd3_compile": true,
+    "recurrence": 1,
+    "gamma": 0.99,
+    "reward_scale": 1.0,
+    "reward_clip": 1000000000.0,
+    "normalize_returns": false,
+    "exploration_loss_coeff": 0.0,
+    "value_loss_coeff": 0.5,
+    "kl_loss_coeff": 0.1,
+    "gae_lambda": 0.95,
+    "ppo_clip_ratio": 0.1,
+    "ppo_clip_value": 0.2,
+    "max_grad_norm": 4.0,
+    "learning_rate": 0.0003,
+    "normalize_input": false,
+    "actor_worker_gpus": [],
+    "train_for_env_steps": 25000000,
+    "save_every_sec": 60,
+    "keep_checkpoints": 5,
+    "encoder_mlp_layers": [
+      64,
+      64
+    ],
+    "use_rnn": false,
+    "actor_critic_share_weights": true,
+    "adaptive_stddev": false,
+    "use_env_info_cache": false,
+    "with_wandb": true,
+    "wandb_project": "latency-sensitive-bench",
+    "wandb_group": "g1-sonic-balance-fasttd3",
+    "wandb_job_type": "sample_factory",
+    "wandb_tags": [
+      "humanoidbench",
+      "g1",
+      "balance",
+      "fasttd3"
+    ],
+    "gym_task_name": "g1_sonic_balance_simple",
+    "gym_env_id": "LatencyBench/G1SonicBalanceSimple-v0",
+    "gym_make_kwargs_json": "{\"render_mode\": null, \"width\": 256, \"height\": 256, \"scene_path\": \"/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_sonic_assets/assets/sonic_g1/envs/scene_balance_simple.xml\", \"control_mode\": \"raw_body_target\", \"target_velocity_ramp_time_s\": 0.1, \"raw_learn_hands\": true}",
+    "gym_registration_imports_json": "[\"latency_bench.envs.humanoidbench_sonic\"]",
+    "gym_action_space_json": "{\"type\": \"box\", \"labels\": [\"body_0\", \"body_1\", \"body_2\", \"body_3\", \"body_4\", \"body_5\", \"body_6\", \"body_7\", \"body_8\", \"body_9\", \"body_10\", \"body_11\", \"body_12\", \"body_13\", \"body_14\", \"body_15\", \"body_16\", \"body_17\", \"body_18\", \"body_19\", \"body_20\", \"body_21\", \"body_22\", \"body_23\", \"body_24\", \"body_25\", \"body_26\", \"body_27\", \"body_28\", \"hand_0\", \"hand_1\", \"hand_2\", \"hand_3\", \"hand_4\", \"hand_5\", \"hand_6\", \"hand_7\", \"hand_8\", \"hand_9\", \"hand_10\", \"hand_11\", \"hand_12\", \"hand_13\"], \"low\": [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0], \"high\": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], \"dtype\": \"float32\"}",
+    "gym_noop_action_json": "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]",
+    "gym_base_prompt": "Keep the G1 humanoid balanced on the support board.",
+    "env_fps": 50.0,
+    "obs_fps": 50.0,
+    "frame_stack": 1,
+    "mode": "train",
+    "latency_type": "zero",
+    "add_latency_info": false,
+    "hold_policy": "hold",
+    "ordering_policy": "issue_order_fifo",
+    "eval_episodes": 1,
+    "eval_parallel_envs": 1,
+    "eval_max_steps": 1000,
+    "eval_deterministic": true,
+    "episode_metrics_path": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints/g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1/episode_metrics.jsonl",
+    "output_dir": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/g1_sonic_balance_simple/raw_ramp_0p1s/train",
+    "wandb_unique_id": "g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1"
+  },
+  "git_hash": "302d1a2c12eacbfb55d0087cf1d4b5c64538f1f8",
+  "git_repo_name": "git@github.com:ZihanWang314/latency-sensitive-bench.git",
+  "eval_env_frameskip": 1,
+  "output_dir": "/pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/g1_sonic_balance_simple/raw_ramp_0p1s/train",
+  "wandb_unique_id": "g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1"
+}

zero-latency/humanoidbench-balance-simple/small-policy/fasttd3-v1/eval.yaml

@@ -0,0 +1,289 @@
+experiment:
+  name: g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1
+  seed: 1
+backend:
+  type: sample_factory
+  algo: FAST_TD3
+  device: gpu
+  train_dir: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints
+  restart_behavior: resume
+  run_mode: eval
+  actor_worker_gpus: []
+  extra_args:
+  - --load_checkpoint_kind
+  - best
+executor:
+  mode: simulated
+env:
+  name: gymnasium
+  task_name: g1_sonic_balance_simple
+  env_id: LatencyBench/G1SonicBalanceSimple-v0
+  registration_imports:
+  - latency_bench.envs.humanoidbench_sonic
+  make_kwargs:
+    render_mode: rgb_array
+    width: 256
+    height: 256
+    scene_path: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_sonic_assets/assets/sonic_g1/envs/scene_balance_simple.xml
+    control_mode: raw_body_target
+    target_velocity_ramp_time_s: 0.1
+    raw_learn_hands: true
+  env_fps: 50.0
+  obs_fps: 50.0
+  frame_stack: 1
+  action_space:
+    type: box
+    labels:
+    - body_0
+    - body_1
+    - body_2
+    - body_3
+    - body_4
+    - body_5
+    - body_6
+    - body_7
+    - body_8
+    - body_9
+    - body_10
+    - body_11
+    - body_12
+    - body_13
+    - body_14
+    - body_15
+    - body_16
+    - body_17
+    - body_18
+    - body_19
+    - body_20
+    - body_21
+    - body_22
+    - body_23
+    - body_24
+    - body_25
+    - body_26
+    - body_27
+    - body_28
+    - hand_0
+    - hand_1
+    - hand_2
+    - hand_3
+    - hand_4
+    - hand_5
+    - hand_6
+    - hand_7
+    - hand_8
+    - hand_9
+    - hand_10
+    - hand_11
+    - hand_12
+    - hand_13
+    low:
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    high:
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    dtype: float32
+  noop_action:
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  base_prompt: Keep the G1 humanoid balanced on the support board.
+latency:
+  method: zero
+  add_latency_info: false
+scheduler:
+  hold_policy: hold
+  ordering_policy: issue_order_fifo
+policy:
+  type: gymnasium_sf
+  checkpoint_kind: best
+  device: cuda
+  deterministic: true
+training:
+  train_for_env_steps: 25000000
+  num_workers: 128
+  num_envs_per_worker: 1
+  worker_num_splits: 1
+  num_policies: 1
+  batch_size: 128
+  rollout: 1
+  recurrence: 1
+  num_epochs: 1
+  num_batches_per_epoch: 1
+  max_policy_lag: 300
+  learning_rate: 0.0003
+  gamma: 0.99
+  gae_lambda: 0.95
+  ppo_clip_ratio: 0.1
+  ppo_clip_value: 0.2
+  value_loss_coeff: 0.5
+  max_grad_norm: 4.0
+  exploration_loss_coeff: 0.0
+  kl_loss_coeff: 0.1
+  async_rl: false
+  batched_sampling: false
+  use_rnn: false
+  encoder_mlp_layers:
+  - 64
+  - 64
+  normalize_input: false
+  normalize_returns: false
+  save_every_sec: 60
+  keep_checkpoints: 5
+  fasttd3_replay_capacity: 6553600
+  fasttd3_replay_batch_size: 32768
+  fasttd3_transitions_per_update: 64
+  fasttd3_compile: true
+  actor_critic_share_weights: true
+  adaptive_stddev: false
+  reward_scale: 1.0
+  reward_clip: 1000000000.0
+  fasttd3_v_min: -250.0
+  fasttd3_v_max: 250.0
+evaluation:
+  eval_episodes: 10
+  eval_parallel_envs: 1
+  eval_max_steps: 1000
+  eval_deterministic: true
+  eval_latency_values:
+  - 0
+logging:
+  output_dir: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/g1_sonic_balance_simple/raw_ramp_0p1s/eval_best_l0
+  video:
+    enabled: true
+    num_bins: 10
+  save_step_records: true
+  save_action_records: true
+  save_latency_records: false
+  wandb_project: latency-sensitive-bench
+  wandb_group: g1-sonic-balance-fasttd3
+  wandb_job_type: sample_factory
+  wandb_name: g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1
+  wandb_tags:
+  - humanoidbench
+  - g1
+  - balance
+  - fasttd3

zero-latency/humanoidbench-balance-simple/small-policy/fasttd3-v1/train.yaml

@@ -0,0 +1,282 @@
+experiment:
+  name: g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1
+  seed: 1
+backend:
+  type: sample_factory
+  algo: FAST_TD3
+  device: gpu
+  train_dir: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/checkpoints
+  restart_behavior: resume
+  run_mode: train
+  actor_worker_gpus: []
+executor:
+  mode: simulated
+env:
+  name: gymnasium
+  task_name: g1_sonic_balance_simple
+  env_id: LatencyBench/G1SonicBalanceSimple-v0
+  registration_imports:
+  - latency_bench.envs.humanoidbench_sonic
+  make_kwargs:
+    render_mode: null
+    width: 256
+    height: 256
+    scene_path: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_sonic_assets/assets/sonic_g1/envs/scene_balance_simple.xml
+    control_mode: raw_body_target
+    target_velocity_ramp_time_s: 0.1
+    raw_learn_hands: true
+  env_fps: 50.0
+  obs_fps: 50.0
+  frame_stack: 1
+  action_space:
+    type: box
+    labels:
+    - body_0
+    - body_1
+    - body_2
+    - body_3
+    - body_4
+    - body_5
+    - body_6
+    - body_7
+    - body_8
+    - body_9
+    - body_10
+    - body_11
+    - body_12
+    - body_13
+    - body_14
+    - body_15
+    - body_16
+    - body_17
+    - body_18
+    - body_19
+    - body_20
+    - body_21
+    - body_22
+    - body_23
+    - body_24
+    - body_25
+    - body_26
+    - body_27
+    - body_28
+    - hand_0
+    - hand_1
+    - hand_2
+    - hand_3
+    - hand_4
+    - hand_5
+    - hand_6
+    - hand_7
+    - hand_8
+    - hand_9
+    - hand_10
+    - hand_11
+    - hand_12
+    - hand_13
+    low:
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    - -1.0
+    high:
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    - 1.0
+    dtype: float32
+  noop_action:
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  - 0.0
+  base_prompt: Keep the G1 humanoid balanced on the support board.
+latency:
+  method: zero
+  add_latency_info: false
+scheduler:
+  hold_policy: hold
+  ordering_policy: issue_order_fifo
+policy:
+  type: random
+training:
+  train_for_env_steps: 25000000
+  num_workers: 128
+  num_envs_per_worker: 1
+  worker_num_splits: 1
+  num_policies: 1
+  batch_size: 128
+  rollout: 1
+  recurrence: 1
+  num_epochs: 1
+  num_batches_per_epoch: 1
+  max_policy_lag: 300
+  learning_rate: 0.0003
+  gamma: 0.99
+  gae_lambda: 0.95
+  ppo_clip_ratio: 0.1
+  ppo_clip_value: 0.2
+  value_loss_coeff: 0.5
+  max_grad_norm: 4.0
+  exploration_loss_coeff: 0.0
+  kl_loss_coeff: 0.1
+  async_rl: false
+  batched_sampling: false
+  use_rnn: false
+  encoder_mlp_layers:
+  - 64
+  - 64
+  normalize_input: false
+  normalize_returns: false
+  save_every_sec: 60
+  keep_checkpoints: 5
+  fasttd3_replay_capacity: 6553600
+  fasttd3_replay_batch_size: 32768
+  fasttd3_transitions_per_update: 64
+  fasttd3_compile: true
+  actor_critic_share_weights: true
+  adaptive_stddev: false
+  reward_scale: 1.0
+  reward_clip: 1000000000.0
+  fasttd3_v_min: -250.0
+  fasttd3_v_max: 250.0
+evaluation:
+  eval_interval_steps: null
+  eval_episodes: 1
+  eval_parallel_envs: 1
+  eval_max_steps: 1000
+  eval_deterministic: true
+logging:
+  output_dir: /pfs/users/lxy/latency-sensitive-bench/results/humanoidbench_g1_balance_control_comparison_fasttd3_25m/g1_sonic_balance_simple/raw_ramp_0p1s/train
+  video:
+    enabled: false
+    num_bins: 1
+  save_step_records: false
+  save_action_records: false
+  save_latency_records: false
+  wandb_project: latency-sensitive-bench
+  wandb_group: g1-sonic-balance-fasttd3
+  wandb_job_type: sample_factory
+  wandb_name: g1_sonic_balance_simple_raw_ramp_0p1s_fasttd3_25000000steps_s1
+  wandb_tags:
+  - humanoidbench
+  - g1
+  - balance
+  - fasttd3

zero-latency/humanoidbench-balance-simple/small-policy/fasttd3-v1/wandb-summary.json

@@ -0,0 +1 @@
+{"charts/episodic_return_raw":970.935244689081,"train/actor_q_loss":-91.11769,"train/critic_loss":1.5522666,"_wandb":{"runtime":6602},"episode_train/latest_raw_return":970.935244689081,"episode_train/rolling_mean_raw_frames":636.17,"charts/episodic_return":970.935244689081,"episode/latency_raw_frames":0,"episode/length":611.21,"episode_train/latest_raw_frames":1000,"episode/terminated":0.69,"episode/timeout":0.31,"episode_train/rolling_mean_latency_raw_frames":0,"train/actor_loss":-91.11769,"episode/return_per_step":0.9356104,"episode_train/latest_latency_raw_frames":0,"episode_train/completed_raw_frames":24947162,"episode_train/rolling_mean_length":636.17,"episode_train/latest_return":970.935244689081,"episode_train/cumulative_mean_length":290.78996631348275,"episode_train/completed_episodes":85791,"global_step":24982528,"episode/truncated":0.31,"train/actor_action_l2":28.506573,"episode_train/rolling_mean_raw_return":607.1002842987373,"episode_train/cumulative_mean_raw_frames":290.78996631348275,"episode_train/latest_length":1000,"episode_train/cumulative_mean_latency_raw_frames":0,"perf/_sample_throughput":3829.307,"_timestamp":1.7878645290728197e+09,"episode/raw_return":582.8459,"episode_train/rolling_mean_return":607.1002842987373,"episode_train/completed_decision_steps":24947162,"episode/raw_frames":611.21,"_step":94472,"episode_train/cumulative_mean_return":269.73134344245017,"_runtime":6602,"episode/index":668.73,"episode_train/cumulative_mean_raw_return":269.73134344245017,"rollout/ep_raw_rew_mean":607.1002842987373,"train/q_value":91.16704,"train/env_steps":0,"episode/return":582.8459,"perf/_fps":3826.034,"rollout/ep_rew_mean":607.1002842987373}