Add h40_40k (RACE2 VLABench, 40k EMA + full serving model, logs, provenance)

h40_40k/README.md

@@ -0,0 +1,31 @@
+# RACE2 H=40, 40k EMA (VLABench)
+
+Config `pi05_vlabench_race2_h40` (model class `Race2PI0`), experiment `h40_40k_s42`.
+Full RACE2 recipe with action horizon 40 (evaluation replan interval 40), part of the action-horizon sweep H5/H10/…/H50. Same teacher, data, schedule and losses as h20_40k/.
+
+| | |
+|---|---|
+| Init | official VLABench pi0.5 teacher (`teacher_official/`, SHA256 `f6762a23…`) |
+| Training | 40,000 steps, seed 42, global batch 64 = 2 GPUs × micro 32 (accumulation 1), warmup 500 → 5e-05 → cosine → 5e-06, AdamW (0.9, 0.95, wd 1e-10, clip 1.0), EMA 0.999, λ_timing 0.05, λ_base_fm 0.1, n_tau 10, sigma 1.0, support 3.0, jitter 0.5, VLM frozen |
+| Data | `SeonghoonYu/vlabench_primitive_ft_lerobot_224` rev `5772f03c…` (5,000 ep), changepoints `vlabench_cps.npz` |
+| Hardware | 2 x NVIDIA H100 80GB HBM3, trained 2026-09-17T16:27Z to 2026-09-18T12:41Z |
+| Final train metrics (step 39,900) | fm 0.0337, bce 0.1307, peak±1 0.5873, pres_acc 0.7672 |
+
+## Evaluation (40k EMA, horizon = replan = 40, seed 7, tracks 1/2/3/4/6, 50 ep/task, physics errors excluded: 19)
+
+| SR | IS | PS | T1 | T2 | T3 | T4 | T6 |
+|---:|---:|---:|---:|---:|---:|---:|---:|
+| **23.98** | 59.89 | 40.85 | 46.12 | 21.71 | 15.53 | 12.87 | 23.65 |
+
+Aggregation: task mean → equal track weight; PhysicsError episodes excluded from the task denominator. Full details in `logs/eval_summary.json`.
+
+## Files
+
+- `model.safetensors` — full serving model: frozen teacher VLM tensors (603) + EMA tensors (264: action expert, projections, RACE timing head, field). Every tensor verified byte-for-byte against its source; SHA256 in `SHA256SUMS`.
+- `overlay/ema.safetensors`, `overlay/metadata.pt` — the slim trainable-only EMA checkpoint written by the trainer (exactly what was evaluated).
+- `assets/vlabench/vlabench_ft_primitive/norm_stats.json` — normalization statistics (identical to the teacher's).
+- `training_config.json` — readable copy of the training config from `metadata.pt`; `provenance.json` — run identity, hashes, hardware, timing, evaluation summary.
+- `logs/` — trainer metric lines, gzipped full trainer log, driver status/history, evaluation log/summary/spec/manifest, smoke log.
+- `source/` — the code this run needs beyond the base repository (presets, ablation model/config/trainer where applicable).
+
+Serve with `scripts/serve_policy.py --policy.config pi05_vlabench_race2_h40 --policy.dir <this folder>` in the matching OpenPI fork; the action horizon comes from the config, not from tensor shapes. Optimizer states are not included.

h40_40k/SHA256SUMS

@@ -0,0 +1,2 @@
+77828b45bd8854be56b61691e40db9e4f92716319191538bc88e46aa9c363c98  model.safetensors
+232d829b3877e4c68725b94b8b3d1ccfa9c70dd8e109e46d74f339ca7dd2cffe  overlay/ema.safetensors

h40_40k/assets/vlabench/vlabench_ft_primitive/norm_stats.json

@@ -0,0 +1,280 @@
+{
+  "norm_stats": {
+    "state": {
+      "mean": [
+        0.02967369742691517,
+        0.39555251598358154,
+        0.25719010829925537,
+        0.4673101305961609,
+        -0.37718722224235535,
+        -0.11985507607460022,
+        0.44498270750045776,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        0.1581457555294037,
+        0.13765743374824524,
+        0.1370578110218048,
+        2.873211145401001,
+        0.625612199306488,
+        1.7478904724121094,
+        0.49696388840675354,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -0.2880732418596744,
+        0.16642616012468936,
+        0.016219121195189656,
+        -3.1415927410125732,
+        -1.5606922270774841,
+        -3.1378225915908815,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        0.33706892974972724,
+        0.7168794604014606,
+        0.4705753956319765,
+        3.1403361039161677,
+        0.029222321224212644,
+        3.1353093175888063,
+        0.9998,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ]
+    },
+    "actions": {
+      "mean": [
+        -0.00008998554403660819,
+        0.004090703558176756,
+        0.0006838542176410556,
+        0.5465309023857117,
+        -0.004646406974643469,
+        -0.019831420853734016,
+        0.5484480857849121,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "std": [
+        0.012184085324406624,
+        0.013969186693429947,
+        0.021805619820952415,
+        3.0321147441864014,
+        0.028920045122504234,
+        0.7395531535148621,
+        0.497647225856781,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q01": [
+        -0.01572922980785374,
+        -0.011965612983703577,
+        -0.012324783468246447,
+        -6.28066982460022,
+        -0.07970612406730648,
+        -1.807010852813721,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ],
+      "q99": [
+        0.014919160246849028,
+        0.016237698739767104,
+        0.020379505300521905,
+        6.280669824600221,
+        0.06261572251319891,
+        1.7844429779052735,
+        0.9998,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0
+      ]
+    }
+  }
+}

h40_40k/logs/commands.json

@@ -0,0 +1,64 @@
+{
+  "method": "race2",
+  "config": "pi05_vlabench_race2_h40",
+  "exp_name": "h40_40k_s42",
+  "horizon": 40,
+  "micro_batch_size": 32,
+  "save_overlays_from_step": 40000,
+  "eval_warmup_timeout": 7200.0,
+  "batch_size": 64,
+  "world_size": 2,
+  "per_gpu_batch_size": 32,
+  "accumulation_steps": 1,
+  "gpus": [
+    4,
+    5
+  ],
+  "train_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun",
+    "--standalone",
+    "--nnodes=1",
+    "--nproc_per_node=2",
+    "scripts/race2/train_race2_vlabench.py",
+    "pi05_vlabench_race2_h40",
+    "--exp_name",
+    "h40_40k_s42",
+    "--model.micro-batch-size",
+    "32",
+    "--save-overlays-from-step",
+    "40000",
+    "--resume"
+  ],
+  "eval_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+    "scripts/vlabench/eval_pool.py",
+    "--method",
+    "race2",
+    "--horizon",
+    "40",
+    "--exp-name",
+    "h40_40k_s42",
+    "--step",
+    "40000",
+    "--gpus",
+    "4,5",
+    "--jobs-per-gpu",
+    "2",
+    "--groups-per-job",
+    "2",
+    "--n-episode",
+    "50",
+    "--warmup-timeout",
+    "7200.0",
+    "--openpi-root",
+    "/home/virl/seonghoon/vlabench/runtime/openpi",
+    "--vlabench-root",
+    "/home/virl/seonghoon/vlabench/VLABench",
+    "--sim-python",
+    "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"
+  ],
+  "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors",
+  "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42",
+  "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi",
+  "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup"
+}

h40_40k/logs/driver.log

@@ -0,0 +1,45 @@
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-17T16:27:48.099349+00:00", "pid": 41572, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "training", "time": "2026-09-17T16:27:48.100679+00:00", "pid": 41572, "child_pid": 41585, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/train.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T03:47:19.476405+00:00", "pid": 41572, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T03:47:19.605267+00:00", "pid": 41572, "child_pid": 264709, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "failed", "time": "2026-09-18T04:30:20.733207+00:00", "pid": 41572, "error": "RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+Traceback (most recent call last):
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 294, in <module>
+    main()
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 277, in main
+    run(spec["eval_command"], "evaluating", "eval.log", eval_env)
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 250, in run
+    raise RuntimeError(f"{stage} exited with code {code}; see {log_dir / logfile}")
+RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log
+Traceback (most recent call last):
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 298, in <module>
+    main()
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 195, in main
+    reports = setup_reports(args)
+              ^^^^^^^^^^^^^^^^^^^
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 179, in setup_reports
+    if not path.is_file():
+           ^^^^^^^^^^^^^^
+  File "/home/virl/.local/share/uv/python/cpython-3.11.13-linux-x86_64-gnu/lib/python3.11/pathlib.py", line 1267, in is_file
+    return S_ISREG(self.stat().st_mode)
+                   ^^^^^^^^^^^
+  File "/home/virl/.local/share/uv/python/cpython-3.11.13-linux-x86_64-gnu/lib/python3.11/pathlib.py", line 1013, in stat
+    return os.stat(self, follow_symlinks=follow_symlinks)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+PermissionError: [Errno 13] Permission denied: '/root/vlabench/race_eval/logs/h30_h35_setup/dataset_validation.json'
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-18T12:30:26.315629+00:00", "pid": 548519, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T12:30:26.325597+00:00", "pid": 548519, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T12:30:26.326791+00:00", "pid": 548519, "child_pid": 548653, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "failed", "time": "2026-09-18T12:30:28.420691+00:00", "pid": 548519, "error": "RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+Traceback (most recent call last):
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 298, in <module>
+    main()
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 281, in main
+    run(spec["eval_command"], "evaluating", "eval.log", eval_env)
+  File "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/run_h25_job.py", line 254, in run
+    raise RuntimeError(f"{stage} exited with code {code}; see {log_dir / logfile}")
+RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-18T12:41:02.604984+00:00", "pid": 556872, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T12:41:02.615624+00:00", "pid": 556872, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T12:41:03.460641+00:00", "pid": 556872, "child_pid": 557238, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "complete", "time": "2026-09-18T19:02:48.718653+00:00", "pid": 556872}

h40_40k/logs/eval.log

@@ -0,0 +1,3 @@
+[2026-09-18 13:30:20] POOL_EVALUATION_FAILED: RuntimeError: Track 4 evaluator exited with status 1
+[2026-09-18 21:30:28] POOL_EVALUATION_FAILED: ValueError: Existing pool_plan.json differs
+[2026-09-19 04:02:48] POOL_EVALUATION_COMPLETE: /home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/summary.json

h40_40k/logs/eval_pool_manifest.json

@@ -0,0 +1,282 @@
+{
+  "attempt": "20260918_214105_db35b9eb0a",
+  "jobs": {
+    "1": {
+      "gpu": 4,
+      "pid": 557344,
+      "command": [
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+        "--pool-worker",
+        "--method",
+        "race2",
+        "--horizon",
+        "40",
+        "--exp-name",
+        "h40_40k_s42",
+        "--step",
+        "40000",
+        "--gpu",
+        "4",
+        "--n-episode",
+        "50",
+        "--groups",
+        "2",
+        "--tracks",
+        "1",
+        "--replan-steps",
+        "40",
+        "--openpi-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi",
+        "--vlabench-root",
+        "/home/virl/seonghoon/vlabench/VLABench",
+        "--checkpoint-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+        "--output-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_1",
+        "--policy-python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "--sim-python",
+        "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+        "--startup-timeout",
+        "1200.0",
+        "--warmup-timeout",
+        "7200.0",
+        "--task-timeout",
+        "21600.0"
+      ],
+      "status": "complete",
+      "started_at": 1789735265.9548361,
+      "env_limits": {
+        "TORCHINDUCTOR_COMPILE_THREADS": "2",
+        "OMP_NUM_THREADS": "1",
+        "OPENBLAS_NUM_THREADS": "1",
+        "MKL_NUM_THREADS": "1"
+      },
+      "finished_at": 1789745260.6259005
+    },
+    "2": {
+      "gpu": 5,
+      "pid": 557345,
+      "command": [
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+        "--pool-worker",
+        "--method",
+        "race2",
+        "--horizon",
+        "40",
+        "--exp-name",
+        "h40_40k_s42",
+        "--step",
+        "40000",
+        "--gpu",
+        "5",
+        "--n-episode",
+        "50",
+        "--groups",
+        "2",
+        "--tracks",
+        "2",
+        "--replan-steps",
+        "40",
+        "--openpi-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi",
+        "--vlabench-root",
+        "/home/virl/seonghoon/vlabench/VLABench",
+        "--checkpoint-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+        "--output-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_2",
+        "--policy-python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "--sim-python",
+        "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+        "--startup-timeout",
+        "1200.0",
+        "--warmup-timeout",
+        "7200.0",
+        "--task-timeout",
+        "21600.0"
+      ],
+      "status": "complete",
+      "started_at": 1789735265.9570465,
+      "env_limits": {
+        "TORCHINDUCTOR_COMPILE_THREADS": "2",
+        "OMP_NUM_THREADS": "1",
+        "OPENBLAS_NUM_THREADS": "1",
+        "MKL_NUM_THREADS": "1"
+      },
+      "finished_at": 1789745687.823386
+    },
+    "3": {
+      "gpu": 4,
+      "pid": 557346,
+      "command": [
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+        "--pool-worker",
+        "--method",
+        "race2",
+        "--horizon",
+        "40",
+        "--exp-name",
+        "h40_40k_s42",
+        "--step",
+        "40000",
+        "--gpu",
+        "4",
+        "--n-episode",
+        "50",
+        "--groups",
+        "2",
+        "--tracks",
+        "3",
+        "--replan-steps",
+        "40",
+        "--openpi-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi",
+        "--vlabench-root",
+        "/home/virl/seonghoon/vlabench/VLABench",
+        "--checkpoint-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+        "--output-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_3",
+        "--policy-python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "--sim-python",
+        "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+        "--startup-timeout",
+        "1200.0",
+        "--warmup-timeout",
+        "7200.0",
+        "--task-timeout",
+        "21600.0"
+      ],
+      "status": "complete",
+      "started_at": 1789735265.9699328,
+      "env_limits": {
+        "TORCHINDUCTOR_COMPILE_THREADS": "2",
+        "OMP_NUM_THREADS": "1",
+        "OPENBLAS_NUM_THREADS": "1",
+        "MKL_NUM_THREADS": "1"
+      },
+      "finished_at": 1789746741.2099245
+    },
+    "4": {
+      "gpu": 5,
+      "pid": 557354,
+      "command": [
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+        "--pool-worker",
+        "--method",
+        "race2",
+        "--horizon",
+        "40",
+        "--exp-name",
+        "h40_40k_s42",
+        "--step",
+        "40000",
+        "--gpu",
+        "5",
+        "--n-episode",
+        "50",
+        "--groups",
+        "2",
+        "--tracks",
+        "4",
+        "--replan-steps",
+        "40",
+        "--openpi-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi",
+        "--vlabench-root",
+        "/home/virl/seonghoon/vlabench/VLABench",
+        "--checkpoint-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+        "--output-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_4",
+        "--policy-python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "--sim-python",
+        "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+        "--startup-timeout",
+        "1200.0",
+        "--warmup-timeout",
+        "7200.0",
+        "--task-timeout",
+        "21600.0"
+      ],
+      "status": "complete",
+      "started_at": 1789735266.2981925,
+      "env_limits": {
+        "TORCHINDUCTOR_COMPILE_THREADS": "2",
+        "OMP_NUM_THREADS": "1",
+        "OPENBLAS_NUM_THREADS": "1",
+        "MKL_NUM_THREADS": "1"
+      },
+      "finished_at": 1789746235.1989994
+    },
+    "6": {
+      "gpu": 4,
+      "pid": 860179,
+      "command": [
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+        "--pool-worker",
+        "--method",
+        "race2",
+        "--horizon",
+        "40",
+        "--exp-name",
+        "h40_40k_s42",
+        "--step",
+        "40000",
+        "--gpu",
+        "4",
+        "--n-episode",
+        "50",
+        "--groups",
+        "2",
+        "--tracks",
+        "6",
+        "--replan-steps",
+        "40",
+        "--openpi-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi",
+        "--vlabench-root",
+        "/home/virl/seonghoon/vlabench/VLABench",
+        "--checkpoint-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+        "--output-root",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_6",
+        "--policy-python",
+        "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+        "--sim-python",
+        "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+        "--startup-timeout",
+        "1200.0",
+        "--warmup-timeout",
+        "7200.0",
+        "--task-timeout",
+        "21600.0"
+      ],
+      "status": "complete",
+      "started_at": 1789745260.6558871,
+      "env_limits": {
+        "TORCHINDUCTOR_COMPILE_THREADS": "2",
+        "OMP_NUM_THREADS": "1",
+        "OPENBLAS_NUM_THREADS": "1",
+        "MKL_NUM_THREADS": "1"
+      },
+      "finished_at": 1789758166.684446
+    }
+  },
+  "prior_manifests": [
+    "logs/20260918_214105_db35b9eb0a/previous_manifest.json"
+  ],
+  "max_active_per_gpu": {
+    "4": 2,
+    "5": 2
+  }
+}

h40_40k/logs/eval_spec.json

@@ -0,0 +1,102 @@
+{
+  "config": "pi05_vlabench_race2_h40",
+  "exp_name": "h40_40k_s42",
+  "step": 40000,
+  "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000",
+  "ema_sha256": "232d829b3877e4c68725b94b8b3d1ccfa9c70dd8e109e46d74f339ca7dd2cffe",
+  "horizon": 40,
+  "replan_steps": 40,
+  "seed": 7,
+  "policy_seed": 7,
+  "policy_noise_rng": "independent_track_servers",
+  "requested_episodes": 50,
+  "tracks": {
+    "track_1_in_distribution": {
+      "select_painting": 50,
+      "select_book": 50,
+      "select_drink": 50,
+      "select_chemistry_tube": 50,
+      "select_poker": 50,
+      "select_mahjong": 50,
+      "select_toy": 50,
+      "select_fruit": 50,
+      "add_condiment": 50,
+      "insert_flower": 50
+    },
+    "track_2_cross_category": {
+      "select_painting": 50,
+      "select_fruit": 50,
+      "select_toy": 50,
+      "select_poker": 50,
+      "select_mahjong": 50,
+      "insert_flower": 10,
+      "select_chemistry_tube": 50,
+      "add_condiment": 50,
+      "select_drink": 50,
+      "select_book": 50
+    },
+    "track_3_common_sense": {
+      "select_painting": 50,
+      "select_book": 50,
+      "select_drink": 50,
+      "select_chemistry_tube": 50,
+      "select_toy": 50,
+      "select_fruit": 50,
+      "add_condiment": 50,
+      "insert_flower": 50,
+      "select_nth_largest_poker": 50,
+      "select_unique_type_mahjong": 50
+    },
+    "track_4_semantic_instruction": {
+      "select_painting": 50,
+      "select_book": 50,
+      "select_drink": 50,
+      "select_chemistry_tube": 50,
+      "select_poker": 50,
+      "select_mahjong": 50,
+      "select_toy": 50,
+      "select_fruit": 50,
+      "add_condiment": 50,
+      "insert_flower": 50
+    },
+    "track_6_unseen_texture": {
+      "select_painting": 50,
+      "select_book": 50,
+      "select_drink": 50,
+      "select_chemistry_tube": 50,
+      "select_poker": 50,
+      "select_mahjong": 50,
+      "select_toy": 50,
+      "select_fruit": 50,
+      "add_condiment": 50,
+      "insert_flower": 50
+    }
+  },
+  "track_sha256": {
+    "track_1_in_distribution": "61b07440c6f4272705bea0c8d721bc196e744232f48408075613cc9b6f7c8c04",
+    "track_2_cross_category": "cf004383de22d994f83126ca30a315cbfc6a92fd0dbb8d25fa4c185dd75a21fb",
+    "track_3_common_sense": "2aaa298580e74bec13555a6950b95241edf6c5264b498eeaf0e919837952ec6d",
+    "track_4_semantic_instruction": "9aff9dfc778ae35f299dce4c3aa07afbdc5275847490abd8037f3f5bdd9de411",
+    "track_6_unseen_texture": "249c1bca941b13267571b7641a91bcfbaffbc9c2b6b210d9ee2d2513324aa817"
+  },
+  "vlabench_root": "/home/virl/seonghoon/vlabench/VLABench",
+  "execution_mode": "pool",
+  "pool_protocol": {
+    "name": "track_pool_v1",
+    "policy_noise_rng": "independent_track_servers",
+    "policy_seed": 7,
+    "jobs_per_gpu": 2,
+    "groups_per_job": 2,
+    "note": "Each track has an independent seed-7 policy server with a shared noise RNG among its task clients. Client ordering and partial resumes can change draws; this is not bit-identical to a single-server or four-shard evaluation."
+  },
+  "gpus": [
+    4,
+    5
+  ],
+  "env_limits": {
+    "TORCHINDUCTOR_COMPILE_THREADS": "2",
+    "OMP_NUM_THREADS": "1",
+    "OPENBLAS_NUM_THREADS": "1",
+    "MKL_NUM_THREADS": "1"
+  }
+}

h40_40k/logs/eval_summary.json

@@ -0,0 +1,1238 @@
+{
+  "status": "complete",
+  "spec": {
+    "config": "pi05_vlabench_race2_h40",
+    "exp_name": "h40_40k_s42",
+    "step": 40000,
+    "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000",
+    "ema_sha256": "232d829b3877e4c68725b94b8b3d1ccfa9c70dd8e109e46d74f339ca7dd2cffe",
+    "horizon": 40,
+    "replan_steps": 40,
+    "seed": 7,
+    "policy_seed": 7,
+    "policy_noise_rng": "independent_track_servers",
+    "requested_episodes": 50,
+    "tracks": {
+      "track_1_in_distribution": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      },
+      "track_2_cross_category": {
+        "select_painting": 50,
+        "select_fruit": 50,
+        "select_toy": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "insert_flower": 10,
+        "select_chemistry_tube": 50,
+        "add_condiment": 50,
+        "select_drink": 50,
+        "select_book": 50
+      },
+      "track_3_common_sense": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50,
+        "select_nth_largest_poker": 50,
+        "select_unique_type_mahjong": 50
+      },
+      "track_4_semantic_instruction": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      },
+      "track_6_unseen_texture": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      }
+    },
+    "track_sha256": {
+      "track_1_in_distribution": "61b07440c6f4272705bea0c8d721bc196e744232f48408075613cc9b6f7c8c04",
+      "track_2_cross_category": "cf004383de22d994f83126ca30a315cbfc6a92fd0dbb8d25fa4c185dd75a21fb",
+      "track_3_common_sense": "2aaa298580e74bec13555a6950b95241edf6c5264b498eeaf0e919837952ec6d",
+      "track_4_semantic_instruction": "9aff9dfc778ae35f299dce4c3aa07afbdc5275847490abd8037f3f5bdd9de411",
+      "track_6_unseen_texture": "249c1bca941b13267571b7641a91bcfbaffbc9c2b6b210d9ee2d2513324aa817"
+    },
+    "vlabench_root": "/home/virl/seonghoon/vlabench/VLABench",
+    "execution_mode": "pool",
+    "pool_protocol": {
+      "name": "track_pool_v1",
+      "policy_noise_rng": "independent_track_servers",
+      "policy_seed": 7,
+      "jobs_per_gpu": 2,
+      "groups_per_job": 2,
+      "note": "Each track has an independent seed-7 policy server with a shared noise RNG among its task clients. Client ordering and partial resumes can change draws; this is not bit-identical to a single-server or four-shard evaluation."
+    },
+    "gpus": [
+      4,
+      5
+    ],
+    "env_limits": {
+      "TORCHINDUCTOR_COMPILE_THREADS": "2",
+      "OMP_NUM_THREADS": "1",
+      "OPENBLAS_NUM_THREADS": "1",
+      "MKL_NUM_THREADS": "1"
+    }
+  },
+  "tracks": {
+    "track_1_in_distribution": {
+      "status": "complete",
+      "track": "track_1_in_distribution",
+      "n_tasks": 10,
+      "episodes_per_task": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      },
+      "total_episodes": 500,
+      "per_task": {
+        "select_painting": {
+          "success_rate": 0.26,
+          "intention_score": 0.58,
+          "progress_score": 0.26
+        },
+        "select_book": {
+          "success_rate": 0.3673469387755102,
+          "intention_score": 0.8367346938775511,
+          "progress_score": 0.5510204081632653
+        },
+        "select_drink": {
+          "success_rate": 0.48,
+          "intention_score": 1.0,
+          "progress_score": 0.66
+        },
+        "select_chemistry_tube": {
+          "success_rate": 0.52,
+          "intention_score": 0.02,
+          "progress_score": 0.74
+        },
+        "select_poker": {
+          "success_rate": 0.54,
+          "intention_score": 0.86,
+          "progress_score": 0.6866666666666668
+        },
+        "select_mahjong": {
+          "success_rate": 0.22448979591836735,
+          "intention_score": 0.8775510204081632,
+          "progress_score": 0.3163265306122449
+        },
+        "select_toy": {
+          "success_rate": 0.64,
+          "intention_score": 0.18,
+          "progress_score": 0.81
+        },
+        "select_fruit": {
+          "success_rate": 0.7,
+          "intention_score": 1.0,
+          "progress_score": 0.85
+        },
+        "add_condiment": {
+          "success_rate": 0.7,
+          "intention_score": 1.0,
+          "progress_score": 0.8466666666666666
+        },
+        "insert_flower": {
+          "success_rate": 0.18,
+          "intention_score": 1.0,
+          "progress_score": 0.59
+        }
+      },
+      "avg_success_rate": 0.46118367346938777,
+      "avg_intention_score": 0.7354285714285714,
+      "avg_progress_score": 0.6310680272108843,
+      "error_policy": "exclude_physics_errors_v1",
+      "per_task_episode_accounting": {
+        "select_painting": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_book": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 49,
+          "excluded_episodes": 1,
+          "metric_denominator": 49,
+          "excluded_episode_ids": [
+            35
+          ]
+        },
+        "select_drink": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_chemistry_tube": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_poker": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_mahjong": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 49,
+          "excluded_episodes": 1,
+          "metric_denominator": 49,
+          "excluded_episode_ids": [
+            17
+          ]
+        },
+        "select_toy": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_fruit": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "add_condiment": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "insert_flower": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        }
+      },
+      "valid_episodes": 498,
+      "excluded_episodes": 2
+    },
+    "track_2_cross_category": {
+      "status": "complete",
+      "track": "track_2_cross_category",
+      "n_tasks": 10,
+      "episodes_per_task": {
+        "select_painting": 50,
+        "select_fruit": 50,
+        "select_toy": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "insert_flower": 10,
+        "select_chemistry_tube": 50,
+        "add_condiment": 50,
+        "select_drink": 50,
+        "select_book": 50
+      },
+      "total_episodes": 460,
+      "per_task": {
+        "select_painting": {
+          "success_rate": 0.34,
+          "intention_score": 0.5,
+          "progress_score": 0.34
+        },
+        "select_fruit": {
+          "success_rate": 0.46,
+          "intention_score": 0.96,
+          "progress_score": 0.69
+        },
+        "select_toy": {
+          "success_rate": 0.48,
+          "intention_score": 0.1,
+          "progress_score": 0.68
+        },
+        "select_poker": {
+          "success_rate": 0.4,
+          "intention_score": 0.78,
+          "progress_score": 0.5533333333333335
+        },
+        "select_mahjong": {
+          "success_rate": 0.22916666666666666,
+          "intention_score": 0.6875,
+          "progress_score": 0.3020833333333333
+        },
+        "insert_flower": {
+          "success_rate": 0.1,
+          "intention_score": 1.0,
+          "progress_score": 0.25
+        },
+        "select_chemistry_tube": {
+          "success_rate": 0.08163265306122448,
+          "intention_score": 0.0,
+          "progress_score": 0.16326530612244897
+        },
+        "add_condiment": {
+          "success_rate": 0.02,
+          "intention_score": 0.3,
+          "progress_score": 0.1
+        },
+        "select_drink": {
+          "success_rate": 0.04,
+          "intention_score": 0.6,
+          "progress_score": 0.18
+        },
+        "select_book": {
+          "success_rate": 0.02,
+          "intention_score": 0.12,
+          "progress_score": 0.07
+        }
+      },
+      "avg_success_rate": 0.21707993197278913,
+      "avg_intention_score": 0.5047499999999999,
+      "avg_progress_score": 0.33286819727891165,
+      "error_policy": "exclude_physics_errors_v1",
+      "per_task_episode_accounting": {
+        "select_painting": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_fruit": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_toy": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_poker": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_mahjong": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 48,
+          "excluded_episodes": 2,
+          "metric_denominator": 48,
+          "excluded_episode_ids": [
+            0,
+            7
+          ]
+        },
+        "insert_flower": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 10,
+          "valid_episodes": 10,
+          "excluded_episodes": 0,
+          "metric_denominator": 10,
+          "excluded_episode_ids": []
+        },
+        "select_chemistry_tube": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 49,
+          "excluded_episodes": 1,
+          "metric_denominator": 49,
+          "excluded_episode_ids": [
+            30
+          ]
+        },
+        "add_condiment": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_drink": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_book": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        }
+      },
+      "valid_episodes": 457,
+      "excluded_episodes": 3
+    },
+    "track_3_common_sense": {
+      "status": "complete",
+      "track": "track_3_common_sense",
+      "n_tasks": 10,
+      "episodes_per_task": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50,
+        "select_nth_largest_poker": 50,
+        "select_unique_type_mahjong": 50
+      },
+      "total_episodes": 500,
+      "per_task": {
+        "select_painting": {
+          "success_rate": 0.12,
+          "intention_score": 0.46,
+          "progress_score": 0.12
+        },
+        "select_book": {
+          "success_rate": 0.2916666666666667,
+          "intention_score": 0.5208333333333334,
+          "progress_score": 0.4166666666666667
+        },
+        "select_drink": {
+          "success_rate": 0.06,
+          "intention_score": 0.78,
+          "progress_score": 0.19
+        },
+        "select_chemistry_tube": {
+          "success_rate": 0.4,
+          "intention_score": 0.0,
+          "progress_score": 0.63
+        },
+        "select_toy": {
+          "success_rate": 0.18,
+          "intention_score": 0.28,
+          "progress_score": 0.5
+        },
+        "select_fruit": {
+          "success_rate": 0.28,
+          "intention_score": 0.94,
+          "progress_score": 0.6
+        },
+        "add_condiment": {
+          "success_rate": 0.04,
+          "intention_score": 0.68,
+          "progress_score": 0.2333333333333334
+        },
+        "insert_flower": {
+          "success_rate": 0.08,
+          "intention_score": 1.0,
+          "progress_score": 0.48
+        },
+        "select_nth_largest_poker": {
+          "success_rate": 0.08,
+          "intention_score": 0.2,
+          "progress_score": 0.10666666666666666
+        },
+        "select_unique_type_mahjong": {
+          "success_rate": 0.021739130434782608,
+          "intention_score": 0.5,
+          "progress_score": 0.11956521739130435
+        }
+      },
+      "avg_success_rate": 0.15534057971014498,
+      "avg_intention_score": 0.5360833333333334,
+      "avg_progress_score": 0.3396231884057971,
+      "error_policy": "exclude_physics_errors_v1",
+      "per_task_episode_accounting": {
+        "select_painting": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_book": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 48,
+          "excluded_episodes": 2,
+          "metric_denominator": 48,
+          "excluded_episode_ids": [
+            28,
+            35
+          ]
+        },
+        "select_drink": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_chemistry_tube": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_toy": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_fruit": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "add_condiment": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "insert_flower": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_nth_largest_poker": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_unique_type_mahjong": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 46,
+          "excluded_episodes": 4,
+          "metric_denominator": 46,
+          "excluded_episode_ids": [
+            6,
+            8,
+            27,
+            30
+          ]
+        }
+      },
+      "valid_episodes": 494,
+      "excluded_episodes": 6
+    },
+    "track_4_semantic_instruction": {
+      "status": "complete",
+      "track": "track_4_semantic_instruction",
+      "n_tasks": 10,
+      "episodes_per_task": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      },
+      "total_episodes": 500,
+      "per_task": {
+        "select_painting": {
+          "success_rate": 0.02,
+          "intention_score": 0.34,
+          "progress_score": 0.02
+        },
+        "select_book": {
+          "success_rate": 0.3,
+          "intention_score": 0.58,
+          "progress_score": 0.44
+        },
+        "select_drink": {
+          "success_rate": 0.06,
+          "intention_score": 0.82,
+          "progress_score": 0.22
+        },
+        "select_chemistry_tube": {
+          "success_rate": 0.22448979591836735,
+          "intention_score": 0.02040816326530612,
+          "progress_score": 0.5408163265306123
+        },
+        "select_poker": {
+          "success_rate": 0.3,
+          "intention_score": 0.76,
+          "progress_score": 0.35999999999999993
+        },
+        "select_mahjong": {
+          "success_rate": 0.022222222222222223,
+          "intention_score": 0.6,
+          "progress_score": 0.14444444444444443
+        },
+        "select_toy": {
+          "success_rate": 0.06,
+          "intention_score": 0.38,
+          "progress_score": 0.42
+        },
+        "select_fruit": {
+          "success_rate": 0.2,
+          "intention_score": 0.9,
+          "progress_score": 0.47
+        },
+        "add_condiment": {
+          "success_rate": 0.08,
+          "intention_score": 0.7,
+          "progress_score": 0.28
+        },
+        "insert_flower": {
+          "success_rate": 0.02,
+          "intention_score": 1.0,
+          "progress_score": 0.4
+        }
+      },
+      "avg_success_rate": 0.12867120181405894,
+      "avg_intention_score": 0.6100408163265306,
+      "avg_progress_score": 0.32952607709750564,
+      "error_policy": "exclude_physics_errors_v1",
+      "per_task_episode_accounting": {
+        "select_painting": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_book": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_drink": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_chemistry_tube": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 49,
+          "excluded_episodes": 1,
+          "metric_denominator": 49,
+          "excluded_episode_ids": [
+            49
+          ]
+        },
+        "select_poker": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_mahjong": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 45,
+          "excluded_episodes": 5,
+          "metric_denominator": 45,
+          "excluded_episode_ids": [
+            3,
+            5,
+            24,
+            33,
+            41
+          ]
+        },
+        "select_toy": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_fruit": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "add_condiment": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "insert_flower": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        }
+      },
+      "valid_episodes": 494,
+      "excluded_episodes": 6
+    },
+    "track_6_unseen_texture": {
+      "status": "complete",
+      "track": "track_6_unseen_texture",
+      "n_tasks": 10,
+      "episodes_per_task": {
+        "select_painting": 50,
+        "select_book": 50,
+        "select_drink": 50,
+        "select_chemistry_tube": 50,
+        "select_poker": 50,
+        "select_mahjong": 50,
+        "select_toy": 50,
+        "select_fruit": 50,
+        "add_condiment": 50,
+        "insert_flower": 50
+      },
+      "total_episodes": 500,
+      "per_task": {
+        "select_painting": {
+          "success_rate": 0.12,
+          "intention_score": 0.56,
+          "progress_score": 0.12
+        },
+        "select_book": {
+          "success_rate": 0.12,
+          "intention_score": 0.5,
+          "progress_score": 0.27
+        },
+        "select_drink": {
+          "success_rate": 0.22,
+          "intention_score": 0.78,
+          "progress_score": 0.32
+        },
+        "select_chemistry_tube": {
+          "success_rate": 0.2,
+          "intention_score": 0.0,
+          "progress_score": 0.43
+        },
+        "select_poker": {
+          "success_rate": 0.36,
+          "intention_score": 0.66,
+          "progress_score": 0.43333333333333335
+        },
+        "select_mahjong": {
+          "success_rate": 0.125,
+          "intention_score": 0.6041666666666666,
+          "progress_score": 0.21875
+        },
+        "select_toy": {
+          "success_rate": 0.28,
+          "intention_score": 0.12,
+          "progress_score": 0.57
+        },
+        "select_fruit": {
+          "success_rate": 0.32,
+          "intention_score": 0.98,
+          "progress_score": 0.52
+        },
+        "add_condiment": {
+          "success_rate": 0.5,
+          "intention_score": 0.94,
+          "progress_score": 0.6933333333333332
+        },
+        "insert_flower": {
+          "success_rate": 0.12,
+          "intention_score": 0.94,
+          "progress_score": 0.52
+        }
+      },
+      "avg_success_rate": 0.23650000000000002,
+      "avg_intention_score": 0.6084166666666666,
+      "avg_progress_score": 0.4095416666666667,
+      "error_policy": "exclude_physics_errors_v1",
+      "per_task_episode_accounting": {
+        "select_painting": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_book": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_drink": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_chemistry_tube": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_poker": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_mahjong": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 48,
+          "excluded_episodes": 2,
+          "metric_denominator": 48,
+          "excluded_episode_ids": [
+            14,
+            39
+          ]
+        },
+        "select_toy": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "select_fruit": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "add_condiment": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        },
+        "insert_flower": {
+          "error_policy": "exclude_physics_errors_v1",
+          "attempted_episodes": 50,
+          "valid_episodes": 50,
+          "excluded_episodes": 0,
+          "metric_denominator": 50,
+          "excluded_episode_ids": []
+        }
+      },
+      "valid_episodes": 498,
+      "excluded_episodes": 2
+    }
+  },
+  "n_tracks": 5,
+  "n_tasks": 50,
+  "total_episodes": 2460,
+  "shards": {
+    "track_1_in_distribution": {
+      "gpu": 4,
+      "output": "shards/track_1/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_9d9b3e45",
+      "summary": "shards/track_1/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_9d9b3e45/summary.json",
+      "completed_tasks": "shards/track_1/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_9d9b3e45/track_1_in_distribution/completed_tasks",
+      "job": {
+        "gpu": 4,
+        "pid": 557344,
+        "command": [
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+          "--pool-worker",
+          "--method",
+          "race2",
+          "--horizon",
+          "40",
+          "--exp-name",
+          "h40_40k_s42",
+          "--step",
+          "40000",
+          "--gpu",
+          "4",
+          "--n-episode",
+          "50",
+          "--groups",
+          "2",
+          "--tracks",
+          "1",
+          "--replan-steps",
+          "40",
+          "--openpi-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi",
+          "--vlabench-root",
+          "/home/virl/seonghoon/vlabench/VLABench",
+          "--checkpoint-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+          "--output-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_1",
+          "--policy-python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "--sim-python",
+          "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+          "--startup-timeout",
+          "1200.0",
+          "--warmup-timeout",
+          "7200.0",
+          "--task-timeout",
+          "21600.0"
+        ],
+        "status": "complete",
+        "started_at": 1789735265.9548361,
+        "env_limits": {
+          "TORCHINDUCTOR_COMPILE_THREADS": "2",
+          "OMP_NUM_THREADS": "1",
+          "OPENBLAS_NUM_THREADS": "1",
+          "MKL_NUM_THREADS": "1"
+        },
+        "finished_at": 1789745260.6259005
+      }
+    },
+    "track_2_cross_category": {
+      "gpu": 5,
+      "output": "shards/track_2/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_baa99750",
+      "summary": "shards/track_2/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_baa99750/summary.json",
+      "completed_tasks": "shards/track_2/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_baa99750/track_2_cross_category/completed_tasks",
+      "job": {
+        "gpu": 5,
+        "pid": 557345,
+        "command": [
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+          "--pool-worker",
+          "--method",
+          "race2",
+          "--horizon",
+          "40",
+          "--exp-name",
+          "h40_40k_s42",
+          "--step",
+          "40000",
+          "--gpu",
+          "5",
+          "--n-episode",
+          "50",
+          "--groups",
+          "2",
+          "--tracks",
+          "2",
+          "--replan-steps",
+          "40",
+          "--openpi-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi",
+          "--vlabench-root",
+          "/home/virl/seonghoon/vlabench/VLABench",
+          "--checkpoint-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+          "--output-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_2",
+          "--policy-python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "--sim-python",
+          "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+          "--startup-timeout",
+          "1200.0",
+          "--warmup-timeout",
+          "7200.0",
+          "--task-timeout",
+          "21600.0"
+        ],
+        "status": "complete",
+        "started_at": 1789735265.9570465,
+        "env_limits": {
+          "TORCHINDUCTOR_COMPILE_THREADS": "2",
+          "OMP_NUM_THREADS": "1",
+          "OPENBLAS_NUM_THREADS": "1",
+          "MKL_NUM_THREADS": "1"
+        },
+        "finished_at": 1789745687.823386
+      }
+    },
+    "track_3_common_sense": {
+      "gpu": 4,
+      "output": "shards/track_3/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_21b01caf",
+      "summary": "shards/track_3/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_21b01caf/summary.json",
+      "completed_tasks": "shards/track_3/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_21b01caf/track_3_common_sense/completed_tasks",
+      "job": {
+        "gpu": 4,
+        "pid": 557346,
+        "command": [
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+          "--pool-worker",
+          "--method",
+          "race2",
+          "--horizon",
+          "40",
+          "--exp-name",
+          "h40_40k_s42",
+          "--step",
+          "40000",
+          "--gpu",
+          "4",
+          "--n-episode",
+          "50",
+          "--groups",
+          "2",
+          "--tracks",
+          "3",
+          "--replan-steps",
+          "40",
+          "--openpi-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi",
+          "--vlabench-root",
+          "/home/virl/seonghoon/vlabench/VLABench",
+          "--checkpoint-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+          "--output-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_3",
+          "--policy-python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "--sim-python",
+          "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+          "--startup-timeout",
+          "1200.0",
+          "--warmup-timeout",
+          "7200.0",
+          "--task-timeout",
+          "21600.0"
+        ],
+        "status": "complete",
+        "started_at": 1789735265.9699328,
+        "env_limits": {
+          "TORCHINDUCTOR_COMPILE_THREADS": "2",
+          "OMP_NUM_THREADS": "1",
+          "OPENBLAS_NUM_THREADS": "1",
+          "MKL_NUM_THREADS": "1"
+        },
+        "finished_at": 1789746741.2099245
+      }
+    },
+    "track_4_semantic_instruction": {
+      "gpu": 5,
+      "output": "shards/track_4/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_5fae63be",
+      "summary": "shards/track_4/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_5fae63be/summary.json",
+      "completed_tasks": "shards/track_4/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_5fae63be/track_4_semantic_instruction/completed_tasks",
+      "job": {
+        "gpu": 5,
+        "pid": 557354,
+        "command": [
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+          "--pool-worker",
+          "--method",
+          "race2",
+          "--horizon",
+          "40",
+          "--exp-name",
+          "h40_40k_s42",
+          "--step",
+          "40000",
+          "--gpu",
+          "5",
+          "--n-episode",
+          "50",
+          "--groups",
+          "2",
+          "--tracks",
+          "4",
+          "--replan-steps",
+          "40",
+          "--openpi-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi",
+          "--vlabench-root",
+          "/home/virl/seonghoon/vlabench/VLABench",
+          "--checkpoint-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+          "--output-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_4",
+          "--policy-python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "--sim-python",
+          "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+          "--startup-timeout",
+          "1200.0",
+          "--warmup-timeout",
+          "7200.0",
+          "--task-timeout",
+          "21600.0"
+        ],
+        "status": "complete",
+        "started_at": 1789735266.2981925,
+        "env_limits": {
+          "TORCHINDUCTOR_COMPILE_THREADS": "2",
+          "OMP_NUM_THREADS": "1",
+          "OPENBLAS_NUM_THREADS": "1",
+          "MKL_NUM_THREADS": "1"
+        },
+        "finished_at": 1789746235.1989994
+      }
+    },
+    "track_6_unseen_texture": {
+      "gpu": 4,
+      "output": "shards/track_6/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_498302e6",
+      "summary": "shards/track_6/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_498302e6/summary.json",
+      "completed_tasks": "shards/track_6/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7_subset_498302e6/track_6_unseen_texture/completed_tasks",
+      "job": {
+        "gpu": 4,
+        "pid": 860179,
+        "command": [
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/scripts/vlabench/eval_h25.py",
+          "--pool-worker",
+          "--method",
+          "race2",
+          "--horizon",
+          "40",
+          "--exp-name",
+          "h40_40k_s42",
+          "--step",
+          "40000",
+          "--gpu",
+          "4",
+          "--n-episode",
+          "50",
+          "--groups",
+          "2",
+          "--tracks",
+          "6",
+          "--replan-steps",
+          "40",
+          "--openpi-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi",
+          "--vlabench-root",
+          "/home/virl/seonghoon/vlabench/VLABench",
+          "--checkpoint-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints",
+          "--output-root",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/evaluate_results/vlabench_h40/race2_h40_h40_40k_s42_40000_ema_replan40_n50_seed7/shards/track_6",
+          "--policy-python",
+          "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+          "--sim-python",
+          "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python",
+          "--startup-timeout",
+          "1200.0",
+          "--warmup-timeout",
+          "7200.0",
+          "--task-timeout",
+          "21600.0"
+        ],
+        "status": "complete",
+        "started_at": 1789745260.6558871,
+        "env_limits": {
+          "TORCHINDUCTOR_COMPILE_THREADS": "2",
+          "OMP_NUM_THREADS": "1",
+          "OPENBLAS_NUM_THREADS": "1",
+          "MKL_NUM_THREADS": "1"
+        },
+        "finished_at": 1789758166.684446
+      }
+    }
+  },
+  "pool_manifest": "pool_manifest.json",
+  "error_policy": "exclude_physics_errors_v1",
+  "valid_episodes": 2441,
+  "excluded_episodes": 19,
+  "aggregation": "Equal weight for five tracks; equal task weight within each track. Recorded PhysicsError episodes are excluded from each task's metric denominator; ordinary unsuccessful rollouts remain included.",
+  "avg_success_rate": 0.23975507739327617,
+  "avg_intention_score": 0.5989438775510203,
+  "avg_progress_score": 0.4085254313319531
+}

h40_40k/logs/history.jsonl

@@ -0,0 +1,13 @@
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-17T16:27:48.099349+00:00", "pid": 41572, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "training", "time": "2026-09-17T16:27:48.100679+00:00", "pid": 41572, "child_pid": 41585, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/train.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T03:47:19.476405+00:00", "pid": 41572, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T03:47:19.605267+00:00", "pid": 41572, "child_pid": 264709, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "failed", "time": "2026-09-18T04:30:20.733207+00:00", "pid": 41572, "error": "RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-18T12:30:26.315629+00:00", "pid": 548519, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T12:30:26.325597+00:00", "pid": 548519, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T12:30:26.326791+00:00", "pid": 548519, "child_pid": 548653, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "failed", "time": "2026-09-18T12:30:28.420691+00:00", "pid": 548519, "error": "RuntimeError: evaluating exited with code 1; see /home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "launch_validated", "time": "2026-09-18T12:41:02.604984+00:00", "pid": 556872, "launch_validation": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/launch_validation.json"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "trained", "time": "2026-09-18T12:41:02.615624+00:00", "pid": 556872, "checkpoint_validation": {"step": 40000, "config": "pi05_vlabench_race2_h40", "ema": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "n_tensors": 264, "horizon": 40, "micro_batch_size": 32}}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "evaluating", "time": "2026-09-18T12:41:03.460641+00:00", "pid": 556872, "child_pid": 557238, "log": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42/eval.log"}
+{"method": "race2", "config": "pi05_vlabench_race2_h40", "exp_name": "h40_40k_s42", "horizon": 40, "micro_batch_size": 32, "save_overlays_from_step": 40000, "eval_warmup_timeout": 7200.0, "batch_size": 64, "world_size": 2, "per_gpu_batch_size": 32, "accumulation_steps": 1, "gpus": [4, 5], "train_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun", "--standalone", "--nnodes=1", "--nproc_per_node=2", "scripts/race2/train_race2_vlabench.py", "pi05_vlabench_race2_h40", "--exp_name", "h40_40k_s42", "--model.micro-batch-size", "32", "--save-overlays-from-step", "40000", "--resume"], "eval_command": ["/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python", "scripts/vlabench/eval_pool.py", "--method", "race2", "--horizon", "40", "--exp-name", "h40_40k_s42", "--step", "40000", "--gpus", "4,5", "--jobs-per-gpu", "2", "--groups-per-job", "2", "--n-episode", "50", "--warmup-timeout", "7200.0", "--openpi-root", "/home/virl/seonghoon/vlabench/runtime/openpi", "--vlabench-root", "/home/virl/seonghoon/vlabench/VLABench", "--sim-python", "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"], "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors", "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42", "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi", "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup", "stage": "complete", "time": "2026-09-18T19:02:48.718653+00:00", "pid": 556872}

h40_40k/logs/launch_validation.json

@@ -0,0 +1,156 @@
+{
+  "status": "verified",
+  "method": "race2",
+  "config": "pi05_vlabench_race2_h40",
+  "exp_name": "h40_40k_s42",
+  "horizon": 40,
+  "micro_batch_size": 32,
+  "save_overlays_from_step": 40000,
+  "eval_warmup_timeout": 7200.0,
+  "batch_size": 64,
+  "world_size": 2,
+  "per_gpu_batch_size": 32,
+  "accumulation_steps": 1,
+  "gpus": [
+    4,
+    5
+  ],
+  "train_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun",
+    "--standalone",
+    "--nnodes=1",
+    "--nproc_per_node=2",
+    "scripts/race2/train_race2_vlabench.py",
+    "pi05_vlabench_race2_h40",
+    "--exp_name",
+    "h40_40k_s42",
+    "--model.micro-batch-size",
+    "32",
+    "--save-overlays-from-step",
+    "40000",
+    "--resume"
+  ],
+  "eval_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+    "scripts/vlabench/eval_pool.py",
+    "--method",
+    "race2",
+    "--horizon",
+    "40",
+    "--exp-name",
+    "h40_40k_s42",
+    "--step",
+    "40000",
+    "--gpus",
+    "4,5",
+    "--jobs-per-gpu",
+    "2",
+    "--groups-per-job",
+    "2",
+    "--n-episode",
+    "50",
+    "--warmup-timeout",
+    "7200.0",
+    "--openpi-root",
+    "/home/virl/seonghoon/vlabench/runtime/openpi",
+    "--vlabench-root",
+    "/home/virl/seonghoon/vlabench/VLABench",
+    "--sim-python",
+    "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"
+  ],
+  "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors",
+  "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42",
+  "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi",
+  "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup",
+  "setup_reports": {
+    "dataset_validation.json": {
+      "path": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup/dataset_validation.json",
+      "sha256": "756efafff23c16ca1d4efcc33088aee2c3a2446ac3ce99c2dc800788cf984358",
+      "status": "verified"
+    },
+    "teacher_validation.json": {
+      "path": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup/teacher_validation.json",
+      "sha256": "d8fa692b7ed306d32a36120b4c8069fc479c23a69034eba11aec111aebbe6feb",
+      "status": "verified"
+    },
+    "cache_ready.json": {
+      "path": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup/cache_ready.json",
+      "sha256": "481dcb9b8b880cfaa9104f2f9d4c2b27bda6c5fc6665aeee3c2b2453a0a33123",
+      "status": "verified"
+    }
+  },
+  "recipe": {
+    "name": "pi05_vlabench_race2_h40",
+    "exp_name": "h40_40k_s42",
+    "pytorch_weight_path": "checkpoints/pi05_vlabench_official/pytorch",
+    "seed": 42,
+    "batch_size": 64,
+    "pytorch_training_precision": "bfloat16",
+    "optimizer": {
+      "b1": 0.9,
+      "b2": 0.95,
+      "eps": 1e-08,
+      "weight_decay": 1e-10,
+      "clip_gradient_norm": 1.0
+    },
+    "lr_schedule": {
+      "warmup_steps": 500,
+      "peak_lr": 5e-05,
+      "decay_steps": 39500,
+      "decay_lr": 5e-06
+    },
+    "ema_decay": 0.999,
+    "model": {
+      "action_dim": 32,
+      "action_horizon": 40,
+      "max_token_len": 200,
+      "dtype": "bfloat16",
+      "paligemma_variant": "gemma_2b",
+      "action_expert_variant": "gemma_300m",
+      "pi05": true,
+      "discrete_state_input": false,
+      "gemma_depth": 18,
+      "sigma": 1.0,
+      "support": 3.0,
+      "jitter_prob": 0.5,
+      "lambda_timing": 0.05,
+      "lambda_base_fm": 0.1,
+      "read_tau": 1.0,
+      "n_tau": 10,
+      "tau_gate_init": 0.0,
+      "tau_gate_lr_mult": 20.0,
+      "timing_proj_dim": 256,
+      "timing_num_heads": 4,
+      "fast_serving": 0,
+      "kv_layer": -1
+    },
+    "data": {
+      "repo_id": "SeonghoonYu/vlabench_primitive_ft_lerobot_224",
+      "assets": {
+        "assets_dir": "./assets/pi05_vlabench",
+        "asset_id": "vlabench/vlabench_ft_primitive"
+      },
+      "phase_labels_path": null,
+      "phase_label_horizon": null,
+      "cps_path": "assets/gauss_timing/vlabench_cps.npz",
+      "gauss_sigma": 1.0,
+      "gauss_support": 3.0,
+      "base_config": {
+        "repo_id": null,
+        "asset_id": null,
+        "use_quantile_norm": false,
+        "action_sequence_keys": [
+          "actions"
+        ],
+        "prompt_from_task": true
+      }
+    }
+  },
+  "gpu_uuids": [
+    "GPU-2d03f219-d151-0b67-79e5-e37792386e4e",
+    "GPU-5ae598d6-ab6f-6174-09e5-ec35abfd095f"
+  ],
+  "hf_lerobot_home": "/home/virl/seonghoon/vlabench/.cache/huggingface/lerobot",
+  "hf_datasets_cache": "/home/virl/seonghoon/vlabench/.cache/huggingface/datasets",
+  "openpi_data_home": "/home/virl/seonghoon/vlabench/.cache/openpi"
+}

h40_40k/logs/smoke_race2_h40.log

@@ -0,0 +1,26 @@
+head kv=cached VLM values (layer -1) params=790,017 | dtype=bfloat16
+WARNING:root:
+The dataset you requested (SeonghoonYu/vlabench_primitive_ft_lerobot_224) is in 2.0 format.
+While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global
+stats instead of per-episode stats. Update your dataset stats to the new format using this command:
+```
+python lerobot/common/datasets/v21/convert_dataset_v20_to_v21.py --repo-id=SeonghoonYu/vlabench_primitive_ft_lerobot_224
+```
+
+If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)
+or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).
+
+WARNING:root:
+The dataset you requested (SeonghoonYu/vlabench_primitive_ft_lerobot_224) is in 2.0 format.
+While current version of LeRobot is backward-compatible with it, the version of your dataset still uses global
+stats instead of per-episode stats. Update your dataset stats to the new format using this command:
+```
+python lerobot/common/datasets/v21/convert_dataset_v20_to_v21.py --repo-id=SeonghoonYu/vlabench_primitive_ft_lerobot_224
+```
+
+If you encounter a problem, contact LeRobot maintainers on [Discord](https://discord.com/invite/s3KuuzsPFb)
+or open an [issue on GitHub](https://github.com/huggingface/lerobot/issues/new/choose).
+
+train: loss 1.7805 fm 1.4255 base_fm 3.21118 bce 0.6773 | grad race_head 9.01e-01 expert 3.73e+01 field_mod 5.81e-02 VLM 0.00e+00
+serve: |a|max 6.508 finite=True | determinism 0.00e+00 | parity vs reference 0.00e+00 | per-call 361 ms (first 358 ms)
+SMOKE_OK

h40_40k/logs/status.json

@@ -0,0 +1,67 @@
+{
+  "method": "race2",
+  "config": "pi05_vlabench_race2_h40",
+  "exp_name": "h40_40k_s42",
+  "horizon": 40,
+  "micro_batch_size": 32,
+  "save_overlays_from_step": 40000,
+  "eval_warmup_timeout": 7200.0,
+  "batch_size": 64,
+  "world_size": 2,
+  "per_gpu_batch_size": 32,
+  "accumulation_steps": 1,
+  "gpus": [
+    4,
+    5
+  ],
+  "train_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/torchrun",
+    "--standalone",
+    "--nnodes=1",
+    "--nproc_per_node=2",
+    "scripts/race2/train_race2_vlabench.py",
+    "pi05_vlabench_race2_h40",
+    "--exp_name",
+    "h40_40k_s42",
+    "--model.micro-batch-size",
+    "32",
+    "--save-overlays-from-step",
+    "40000",
+    "--resume"
+  ],
+  "eval_command": [
+    "/home/virl/seonghoon/vlabench/runtime/openpi/.venv/bin/python",
+    "scripts/vlabench/eval_pool.py",
+    "--method",
+    "race2",
+    "--horizon",
+    "40",
+    "--exp-name",
+    "h40_40k_s42",
+    "--step",
+    "40000",
+    "--gpus",
+    "4,5",
+    "--jobs-per-gpu",
+    "2",
+    "--groups-per-job",
+    "2",
+    "--n-episode",
+    "50",
+    "--warmup-timeout",
+    "7200.0",
+    "--openpi-root",
+    "/home/virl/seonghoon/vlabench/runtime/openpi",
+    "--vlabench-root",
+    "/home/virl/seonghoon/vlabench/VLABench",
+    "--sim-python",
+    "/home/virl/seonghoon/vlabench/runtime/sim-env/bin/python"
+  ],
+  "checkpoint": "/home/virl/seonghoon/vlabench/runtime/openpi/checkpoints/pi05_vlabench_race2_h40/h40_40k_s42/40000/ema.safetensors",
+  "log_dir": "/home/virl/seonghoon/vlabench/runtime/openpi/logs/h40/race2_h40_40k_s42",
+  "openpi_root": "/home/virl/seonghoon/vlabench/runtime/openpi",
+  "setup_root": "/home/virl/seonghoon/vlabench/race_eval/logs/h30_h35_setup",
+  "stage": "complete",
+  "time": "2026-09-18T19:02:48.718653+00:00",
+  "pid": 556872
+}

h40_40k/logs/train.log.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b7474d4ac3c3f510ccf3529fb28836d276d8217db6855bf3327945d73b5c86c
+size 1697531

h40_40k/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77828b45bd8854be56b61691e40db9e4f92716319191538bc88e46aa9c363c98
+size 7939730180

h40_40k/overlay/ema.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:232d829b3877e4c68725b94b8b3d1ccfa9c70dd8e109e46d74f339ca7dd2cffe
+size 2092955412

h40_40k/overlay/metadata.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e596f728d43a42835ed40edbd93d407eac874029f676858b51ffff8c3f1e5078
+size 3635

h40_40k/provenance.json

@@ -0,0 +1,64 @@
+{
+  "method": "RACE2",
+  "config": "pi05_vlabench_race2_h40",
+  "model_class": "Race2PI0",
+  "exp_name": "h40_40k_s42",
+  "step": 40000,
+  "weights": "EMA",
+  "ema_decay": 0.999,
+  "action_horizon": 40,
+  "training_seed": 42,
+  "world_size": 2,
+  "micro_batch_size": 32,
+  "accumulation_steps": 1,
+  "global_batch": 64,
+  "save_overlays_from_step": 40000,
+  "ema_overlay_sha256": "232d829b3877e4c68725b94b8b3d1ccfa9c70dd8e109e46d74f339ca7dd2cffe",
+  "materialized_model_sha256": "77828b45bd8854be56b61691e40db9e4f92716319191538bc88e46aa9c363c98",
+  "materialization": {
+    "bytes": 7939730180,
+    "frozen_tensors": 603,
+    "ema_tensors": 264,
+    "all_tensor_bytes_match_sources": true
+  },
+  "teacher_sha256": "f6762a23e4374bcbd8f3a7297f159b02d253a335f125eb93595d431d37607172",
+  "teacher_repo": "SeonghoonYu/RACE2_vlabench",
+  "teacher_path": "teacher_official/model.safetensors",
+  "dataset": {
+    "repo_id": "SeonghoonYu/vlabench_primitive_ft_lerobot_224",
+    "revision": "5772f03c3af8df38ee75d8f71cab0075563ed12a"
+  },
+  "changepoints": "assets/gauss_timing/vlabench_cps.npz",
+  "normalization_sha256": "bce33fc1831830d953ef9bc57e520c02ee44e02b7ad07f27ce92b931a879384f",
+  "code_repository": "https://github.com/Seonghoon-Yu/vlabench",
+  "code_revision": "6f4e6e42e8382c20a9958883d68c2144f5daa4e0 (+ uncommitted 2026-09-17/20 changes; the preset is in source/)",
+  "hardware": "2 x NVIDIA H100 80GB HBM3",
+  "trained": "2026-09-17T16:27Z to 2026-09-18T12:41Z",
+  "evaluated": "2026-09-18T12:41Z to 2026-09-18T19:02Z",
+  "evaluation": {
+    "replan_steps": 40,
+    "seed": 7,
+    "tracks": [
+      1,
+      2,
+      3,
+      4,
+      6
+    ],
+    "n_episode": 2460,
+    "valid_episodes": 2441,
+    "physics_errors_excluded": 19,
+    "success_rate_pct": 23.98,
+    "intention_score_pct": 59.89,
+    "progress_score_pct": 40.85,
+    "track_sr_pct": {
+      "T1": 46.12,
+      "T2": 21.71,
+      "T3": 15.53,
+      "T4": 12.87,
+      "T6": 23.65
+    },
+    "policy_noise_rng": "independent_track_servers"
+  },
+  "optimizer_state_included": false
+}

h40_40k/training_config.json

@@ -0,0 +1,106 @@
+{
+  "global_step": 40000,
+  "config": {
+    "name": "pi05_vlabench_race2_h40",
+    "project_name": "openpi",
+    "exp_name": "h40_40k_s42",
+    "model": {
+      "action_dim": 32,
+      "action_horizon": 40,
+      "max_token_len": 200,
+      "dtype": "bfloat16",
+      "paligemma_variant": "gemma_2b",
+      "action_expert_variant": "gemma_300m",
+      "pi05": true,
+      "discrete_state_input": false,
+      "gemma_depth": 18,
+      "sigma": 1.0,
+      "support": 3.0,
+      "jitter_prob": 0.5,
+      "lambda_timing": 0.05,
+      "lambda_base_fm": 0.1,
+      "read_tau": 1.0,
+      "n_tau": 10,
+      "tau_gate_init": 0.0,
+      "tau_gate_lr_mult": 20.0,
+      "timing_proj_dim": 256,
+      "timing_num_heads": 4,
+      "micro_batch_size": 32,
+      "fast_serving": 0,
+      "kv_layer": -1
+    },
+    "weight_loader": {},
+    "pytorch_weight_path": "checkpoints/pi05_vlabench_official/pytorch",
+    "pytorch_weight_path_teacher": null,
+    "pytorch_training_precision": "bfloat16",
+    "lr_schedule": {
+      "warmup_steps": 500,
+      "peak_lr": 5e-05,
+      "decay_steps": 39500,
+      "decay_lr": 5e-06
+    },
+    "optimizer": {
+      "b1": 0.9,
+      "b2": 0.95,
+      "eps": 1e-08,
+      "weight_decay": 1e-10,
+      "clip_gradient_norm": 1.0
+    },
+    "ema_decay": 0.999,
+    "freeze_filter": "Nothing()",
+    "data": {
+      "repo_id": "SeonghoonYu/vlabench_primitive_ft_lerobot_224",
+      "assets": {
+        "assets_dir": "./assets/pi05_vlabench",
+        "asset_id": "vlabench/vlabench_ft_primitive"
+      },
+      "base_config": {
+        "repo_id": null,
+        "asset_id": null,
+        "norm_stats": null,
+        "repack_transforms": {
+          "inputs": [],
+          "outputs": []
+        },
+        "data_transforms": {
+          "inputs": [],
+          "outputs": []
+        },
+        "model_transforms": {
+          "inputs": [],
+          "outputs": []
+        },
+        "use_quantile_norm": false,
+        "action_sequence_keys": [
+          "actions"
+        ],
+        "prompt_from_task": true,
+        "rlds_data_dir": null,
+        "action_space": null,
+        "datasets": []
+      },
+      "phase_labels_path": null,
+      "phase_label_horizon": null,
+      "cps_path": "assets/gauss_timing/vlabench_cps.npz",
+      "gauss_sigma": 1.0,
+      "gauss_support": 3.0
+    },
+    "assets_base_dir": "./assets",
+    "checkpoint_base_dir": "./checkpoints",
+    "seed": 42,
+    "batch_size": 64,
+    "num_workers": 8,
+    "num_train_steps": 40000,
+    "log_interval": 100,
+    "save_interval": 1000,
+    "save_overlays_from_step": 40000,
+    "keep_period": 5000,
+    "overwrite": false,
+    "resume": false,
+    "wandb_enabled": false,
+    "policy_metadata": null,
+    "fsdp_devices": 1
+  },
+  "ema_decay": 0.999,
+  "checkpoint_format": "trainable-only overlays; materialise with scripts/race/materialize_ckpt.py"
+}