Hugging Face's logo

MLL-Lab
/
LAGEN-models

Robotics
PyTorch
Safetensors
vision-language-action
latency-sensitive-bench
multiple-checkpoints
Model card Files
xet
 Community
1
Caesarrr commited on
Commit
092da3f
·
verified ·
1 Parent(s): b037609

Publish atlantis qwengr00t H8 evaluation artifacts

Browse files
Files changed (7) hide show
  1. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/README.md +2 -1
  2. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/eval_l0.json +216 -0
  3. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/action_audit.json +133 -0
  4. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/episode_metrics.jsonl +0 -0
  5. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/eval_config.yaml +145 -0
  6. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/provenance.json +46 -0
  7. zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/publication_status.json +3 -1
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/README.md CHANGED
@@ -4,4 +4,5 @@ Current RGB and eight binary decision-history windows; action horizon one. Initi
4
 
5
  - Training: 5000 steps, seed 42, global batch 128
6
  - W&B: https://wandb.ai/saberrr-zju/starvla_tasks/runs/60bc68a7
7
- - Formal zero-latency evaluation: pending.
 
 
4
 
5
  - Training: 5000 steps, seed 42, global batch 128
6
  - W&B: https://wandb.ai/saberrr-zju/starvla_tasks/runs/60bc68a7
7
+ - Evaluation: 100 episodes, seeds 1000000–1000099, GPU simulator, parallel 32, capacity 1, 60/15 Hz, 7200 raw-frame cap, zero latency
8
+ - Mean raw return: 39662.0
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/eval_l0.json ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_path": "/home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/atlantis-qwengr00t-hist8-zero-s42-v1/checkpoints/model.pt",
3
+ "experiment_name": "atlantis-qwengr00t-hist8-zero-s42-v1-step5000-history01-zero-par32",
4
+ "latency": 0,
5
+ "latency_type": "zero",
6
+ "lengths": [
7
+ 7200,
8
+ 7200,
9
+ 7200,
10
+ 7200,
11
+ 7200,
12
+ 7200,
13
+ 7200,
14
+ 7200,
15
+ 7200,
16
+ 7200,
17
+ 7200,
18
+ 7200,
19
+ 7200,
20
+ 7200,
21
+ 7200,
22
+ 7200,
23
+ 7200,
24
+ 7200,
25
+ 7200,
26
+ 7200,
27
+ 7200,
28
+ 7200,
29
+ 7200,
30
+ 7200,
31
+ 7200,
32
+ 7200,
33
+ 7200,
34
+ 7200,
35
+ 7200,
36
+ 7200,
37
+ 7200,
38
+ 7200,
39
+ 7200,
40
+ 7200,
41
+ 7200,
42
+ 7200,
43
+ 7200,
44
+ 7200,
45
+ 7200,
46
+ 7200,
47
+ 7200,
48
+ 7200,
49
+ 7200,
50
+ 7200,
51
+ 7200,
52
+ 7200,
53
+ 7200,
54
+ 7200,
55
+ 7200,
56
+ 7200,
57
+ 7200,
58
+ 7200,
59
+ 7200,
60
+ 7200,
61
+ 7200,
62
+ 7200,
63
+ 7200,
64
+ 7200,
65
+ 7200,
66
+ 7200,
67
+ 7200,
68
+ 7200,
69
+ 7200,
70
+ 7200,
71
+ 7200,
72
+ 7200,
73
+ 7200,
74
+ 7200,
75
+ 7200,
76
+ 7200,
77
+ 7200,
78
+ 7200,
79
+ 7200,
80
+ 7200,
81
+ 7200,
82
+ 7200,
83
+ 7200,
84
+ 7200,
85
+ 7200,
86
+ 7200,
87
+ 7200,
88
+ 7200,
89
+ 7200,
90
+ 7200,
91
+ 7200,
92
+ 7200,
93
+ 7200,
94
+ 7200,
95
+ 7200,
96
+ 7200,
97
+ 7200,
98
+ 7200,
99
+ 7200,
100
+ 7200,
101
+ 7200,
102
+ 7200,
103
+ 7200,
104
+ 7200,
105
+ 7200,
106
+ 7200
107
+ ],
108
+ "mean_length": 7200.0,
109
+ "mean_return": 39662.0,
110
+ "returns": [
111
+ 39100.0,
112
+ 42800.0,
113
+ 42500.0,
114
+ 43200.0,
115
+ 37500.0,
116
+ 37500.0,
117
+ 36800.0,
118
+ 42500.0,
119
+ 42800.0,
120
+ 43200.0,
121
+ 43200.0,
122
+ 33100.0,
123
+ 36800.0,
124
+ 42500.0,
125
+ 42500.0,
126
+ 42800.0,
127
+ 36800.0,
128
+ 37500.0,
129
+ 37500.0,
130
+ 43200.0,
131
+ 36800.0,
132
+ 37500.0,
133
+ 37500.0,
134
+ 42500.0,
135
+ 36800.0,
136
+ 43200.0,
137
+ 37500.0,
138
+ 36800.0,
139
+ 36800.0,
140
+ 42500.0,
141
+ 36800.0,
142
+ 37500.0,
143
+ 38900.0,
144
+ 36800.0,
145
+ 42800.0,
146
+ 42500.0,
147
+ 37500.0,
148
+ 42800.0,
149
+ 36800.0,
150
+ 37500.0,
151
+ 42800.0,
152
+ 36800.0,
153
+ 37500.0,
154
+ 43200.0,
155
+ 37500.0,
156
+ 37500.0,
157
+ 42500.0,
158
+ 42500.0,
159
+ 42500.0,
160
+ 37500.0,
161
+ 42800.0,
162
+ 36500.0,
163
+ 42800.0,
164
+ 37500.0,
165
+ 37500.0,
166
+ 43200.0,
167
+ 42500.0,
168
+ 36800.0,
169
+ 36800.0,
170
+ 37500.0,
171
+ 42800.0,
172
+ 42500.0,
173
+ 37500.0,
174
+ 37500.0,
175
+ 37500.0,
176
+ 37500.0,
177
+ 42400.0,
178
+ 42500.0,
179
+ 37500.0,
180
+ 37500.0,
181
+ 42500.0,
182
+ 42800.0,
183
+ 43200.0,
184
+ 44700.0,
185
+ 36800.0,
186
+ 37500.0,
187
+ 36800.0,
188
+ 43000.0,
189
+ 43200.0,
190
+ 36800.0,
191
+ 37500.0,
192
+ 36800.0,
193
+ 42500.0,
194
+ 36800.0,
195
+ 42500.0,
196
+ 36800.0,
197
+ 42500.0,
198
+ 42500.0,
199
+ 42500.0,
200
+ 36800.0,
201
+ 37500.0,
202
+ 42500.0,
203
+ 40700.0,
204
+ 42500.0,
205
+ 36800.0,
206
+ 37500.0,
207
+ 42800.0,
208
+ 36800.0,
209
+ 36800.0,
210
+ 36800.0
211
+ ],
212
+ "seed": 1000000,
213
+ "std_return": 2849.483461962887,
214
+ "suite_name": "fixed_0",
215
+ "timestamp_utc": "2026-09-24T11:15:47.943321+00:00"
216
+ }
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/action_audit.json ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval_protocol_version": "hist8-history01-sf-render-v4",
3
+ "history_encoding": "binary_0_1",
4
+ "episodes": 100,
5
+ "episode_seeds": [
6
+ 1000000,
7
+ 1000001,
8
+ 1000002,
9
+ 1000003,
10
+ 1000004,
11
+ 1000005,
12
+ 1000006,
13
+ 1000007,
14
+ 1000008,
15
+ 1000009,
16
+ 1000010,
17
+ 1000011,
18
+ 1000012,
19
+ 1000013,
20
+ 1000014,
21
+ 1000015,
22
+ 1000016,
23
+ 1000017,
24
+ 1000018,
25
+ 1000019,
26
+ 1000020,
27
+ 1000021,
28
+ 1000022,
29
+ 1000023,
30
+ 1000024,
31
+ 1000025,
32
+ 1000026,
33
+ 1000027,
34
+ 1000028,
35
+ 1000029,
36
+ 1000030,
37
+ 1000031,
38
+ 1000032,
39
+ 1000033,
40
+ 1000034,
41
+ 1000035,
42
+ 1000036,
43
+ 1000037,
44
+ 1000038,
45
+ 1000039,
46
+ 1000040,
47
+ 1000041,
48
+ 1000042,
49
+ 1000043,
50
+ 1000044,
51
+ 1000045,
52
+ 1000046,
53
+ 1000047,
54
+ 1000048,
55
+ 1000049,
56
+ 1000050,
57
+ 1000051,
58
+ 1000052,
59
+ 1000053,
60
+ 1000054,
61
+ 1000055,
62
+ 1000056,
63
+ 1000057,
64
+ 1000058,
65
+ 1000059,
66
+ 1000060,
67
+ 1000061,
68
+ 1000062,
69
+ 1000063,
70
+ 1000064,
71
+ 1000065,
72
+ 1000066,
73
+ 1000067,
74
+ 1000068,
75
+ 1000069,
76
+ 1000070,
77
+ 1000071,
78
+ 1000072,
79
+ 1000073,
80
+ 1000074,
81
+ 1000075,
82
+ 1000076,
83
+ 1000077,
84
+ 1000078,
85
+ 1000079,
86
+ 1000080,
87
+ 1000081,
88
+ 1000082,
89
+ 1000083,
90
+ 1000084,
91
+ 1000085,
92
+ 1000086,
93
+ 1000087,
94
+ 1000088,
95
+ 1000089,
96
+ 1000090,
97
+ 1000091,
98
+ 1000092,
99
+ 1000093,
100
+ 1000094,
101
+ 1000095,
102
+ 1000096,
103
+ 1000097,
104
+ 1000098,
105
+ 1000099
106
+ ],
107
+ "issued_actions": 180000,
108
+ "action_counts": {
109
+ "0": 58729,
110
+ "2": 51845,
111
+ "3": 32048,
112
+ "1": 37378
113
+ },
114
+ "largest_action_fraction": 0.32627222222222224,
115
+ "raw_score_min": [
116
+ -0.09850892424583435,
117
+ -0.1825193166732788,
118
+ -0.18227225542068481,
119
+ -0.49613189697265625
120
+ ],
121
+ "raw_score_max": [
122
+ 1.1174542903900146,
123
+ 1.10590398311615,
124
+ 1.1816869974136353,
125
+ 1.0849536657333374
126
+ ],
127
+ "raw_score_mean": [
128
+ 0.32900199618344067,
129
+ 0.21200553999988964,
130
+ 0.2875584516548888,
131
+ 0.1787945480706416
132
+ ]
133
+ }
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/episode_metrics.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/eval_config.yaml ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ experiment:
2
+ name: atlantis-qwengr00t-hist8-zero-s42-v1-step5000-history01-zero-par32
3
+ seed: 1000000
4
+ backend:
5
+ type: sample_factory
6
+ algo: APPO
7
+ device: cuda
8
+ train_dir: results/sample_factory
9
+ restart_behavior: resume
10
+ run_mode: eval
11
+ extra_args:
12
+ - --encoder_conv_architecture
13
+ - convnet_simple
14
+ executor:
15
+ mode: simulated
16
+ simulated_worker_capacity: 1
17
+ simulated_inference_pool: true
18
+ inference_devices:
19
+ - cuda:0
20
+ - cuda:1
21
+ inference_batch_size: 16
22
+ env:
23
+ name: gymnasium
24
+ task_name: atlantis
25
+ env_id: LatencyBench/Atlantis-v0
26
+ registration_imports:
27
+ - latency_bench.envs.gymnasium_atlantis
28
+ make_kwargs:
29
+ base_env_id: ALE/Atlantis-v5
30
+ render_mode: rgb_array
31
+ screen_size: 84
32
+ noop_max: 30
33
+ base_make_kwargs:
34
+ obs_type: rgb
35
+ frameskip: 1
36
+ repeat_action_probability: 0.0
37
+ full_action_space: false
38
+ mode: 0
39
+ difficulty: 0
40
+ max_num_frames_per_episode: 108000
41
+ env_fps: 60
42
+ obs_fps: 15
43
+ frame_stack: 1
44
+ simulator: gpu
45
+ terminal_on_life_loss: false
46
+ clip_reward: false
47
+ noop_action: noop
48
+ action_map:
49
+ noop: 0
50
+ fire: 1
51
+ rightfire: 2
52
+ leftfire: 3
53
+ action_order:
54
+ - noop
55
+ - fire
56
+ - rightfire
57
+ - leftfire
58
+ base_prompt: 'Defend Atlantis by firing at descending enemies. Choose exactly one
59
+ action from: noop, fire, rightfire, leftfire.'
60
+ obs_resize:
61
+ - 224
62
+ - 224
63
+ gpu_env_device: auto
64
+ noop_max: 30
65
+ action_history_decisions: 8
66
+ latency:
67
+ method: zero
68
+ scheduler:
69
+ hold_policy: hold
70
+ ordering_policy: issue_order_fifo
71
+ policy:
72
+ type: starvla
73
+ actions:
74
+ - noop
75
+ - fire
76
+ - rightfire
77
+ - leftfire
78
+ checkpoint_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/atlantis-qwengr00t-hist8-zero-s42-v1/checkpoints/model.pt
79
+ model_config_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/atlantis-qwengr00t-hist8-zero-s42-v1/config.yaml
80
+ task_manifest_path: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/bundles/atlantis-qwengr00t-hist8-zero-s42-v1/manifest.json
81
+ device: cuda:0
82
+ unnorm_key: new_embodiment
83
+ prompt_mode: latency_neutral
84
+ backbone_path: /home/ubuntu/.cache/huggingface/hub/models--Qwen--Qwen3-VL-4B-Instruct/snapshots/ebb281ec70b05090aa6165b016eac8ec08e71b17
85
+ worker_python_executable: /home/ubuntu/miniconda3/envs/starvla_rl_games_openvla/bin/python3.10
86
+ state_source: transport
87
+ training:
88
+ train_for_env_steps: 1000000
89
+ num_workers: 4
90
+ num_envs_per_worker: 16
91
+ worker_num_splits: 1
92
+ num_policies: 1
93
+ batch_size: 1024
94
+ rollout: 64
95
+ recurrence: 32
96
+ num_epochs: 2
97
+ num_batches_per_epoch: 8
98
+ max_policy_lag: 300
99
+ learning_rate: 0.000303
100
+ lr_schedule: constant
101
+ nonlinearity: elu
102
+ gamma: 0.99
103
+ gae_lambda: 0.95
104
+ ppo_clip_ratio: 0.1
105
+ ppo_clip_value: 0.2
106
+ exploration_loss: entropy
107
+ exploration_loss_coeff: 0.003
108
+ value_loss_coeff: 0.5
109
+ max_grad_norm: 0.0
110
+ adam_eps: 1.0e-06
111
+ obs_scale: 1.0
112
+ adaptive_stddev: false
113
+ with_vtrace: false
114
+ async_rl: true
115
+ use_rnn: false
116
+ normalize_input: true
117
+ normalize_returns: true
118
+ stats_avg: 100
119
+ experiment_summaries_interval: 1
120
+ save_every_sec: 600
121
+ keep_checkpoints: 5
122
+ evaluation:
123
+ eval_interval_steps: null
124
+ eval_episodes: 100
125
+ eval_parallel_envs: 32
126
+ eval_max_steps: 7200
127
+ eval_deterministic: true
128
+ eval_suites:
129
+ fixed: []
130
+ normal: []
131
+ uniform: []
132
+ latency_bench_env_backend: atlantis_gpu_batched
133
+ eval_raw_reward: true
134
+ logging:
135
+ output_dir: /home/ubuntu/codex/atari-hist8-20260924/outputs/tasks/atari_hist8_v1/evals/atlantis-qwengr00t-hist8-zero-s42-v1/zero_to_zero
136
+ video:
137
+ enabled: false
138
+ num_bins: 1
139
+ save_step_records: true
140
+ save_action_records: true
141
+ save_latency_records: true
142
+ wandb_project: null
143
+ wandb_group: null
144
+ wandb_job_type: null
145
+ wandb_tags: ''
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/evaluation/provenance.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "collection": {
3
+ "episodes": 200,
4
+ "filter_preset": "none",
5
+ "history": "decision_v1",
6
+ "history_decisions": 8,
7
+ "history_source": "teacher_transport",
8
+ "max_episode_raw_frames": 7200,
9
+ "rollout_overrides": {
10
+ "env_fps": 60,
11
+ "fixed_latency_ms": 0.0,
12
+ "latency_type": "fixed",
13
+ "obs_fps": 15,
14
+ "seed": 42
15
+ },
16
+ "student_image_frames": 1,
17
+ "teacher_checkpoint": "best_000026560_13598720_reward_675769.000.pth",
18
+ "teacher_path": "zero-latency/atlantis/small-policy/sample-factory-hist8-cpu15m-s0-v1",
19
+ "teacher_repo": "latency-sensitive-bench/benchmark-models",
20
+ "teacher_revision": "dfaeecb84cd6b5dfcc3adc0935d0ae86d0b504d2",
21
+ "teacher_sha256": "d41d728d8c2329d3623eb9e01bd9f3f2df4fc0d4f6f2c1f799c4badc24af78c5",
22
+ "train_labels": 324000,
23
+ "training_latency_condition": "zero"
24
+ },
25
+ "training_run_id": "atlantis-qwengr00t-hist8-zero-s42-v1",
26
+ "training_step": 5000,
27
+ "global_batch": 128,
28
+ "base_model": "Qwen/Qwen3-VL-4B-Instruct",
29
+ "wandb_run_id": "60bc68a7",
30
+ "checkpoint_sha256": "dc7081c898fd8324d5c9aa892b5221ad7388a83428cc3508c67b30fc71d0761a",
31
+ "eval_protocol_version": "hist8-history01-sf-render-v4",
32
+ "history_encoding": "binary_0_1",
33
+ "episodes": 100,
34
+ "env_condition": "zero",
35
+ "parallel_envs": 32,
36
+ "max_episode_raw_frames": 7200,
37
+ "inference_devices": [
38
+ "cuda:0",
39
+ "cuda:1"
40
+ ],
41
+ "episode_seeds": [
42
+ 1000000,
43
+ 1000099
44
+ ],
45
+ "profile_sha256": null
46
+ }
zero-latency/atlantis/vla/starvla-qwengr00t-h1/atlantis-qwengr00t-hist8-zero-s42-v1/publication_status.json CHANGED
@@ -1,4 +1,6 @@
1
  {
 
 
2
  "training_status": "complete",
3
- "zero_evaluation_status": "pending"
4
  }
 
1
  {
2
+ "episodes": 100,
3
+ "mean_return": 39662.0,
4
  "training_status": "complete",
5
+ "zero_evaluation_status": "complete"
6
  }