yangfy0627 commited on
Commit
6eebaf6
·
verified ·
1 Parent(s): 41ef0de

Archive complete RL run 15e00de4 with replay, checkpoints, diagnostics and provenance

Browse files
Files changed (28) hide show
  1. RL/Transport50/QC/diverse50/seed_2/best/source_metadata.json +27 -0
  2. RL/Transport50/QC/diverse50/seed_2/best/source_resolved_config.json +1127 -0
  3. RL/Transport50/QC/diverse50/seed_2/best/state.pkl +3 -0
  4. RL/Transport50/QC/diverse50/seed_2/comparison_evaluations.jsonl +81 -0
  5. RL/Transport50/QC/diverse50/seed_2/complete/README.md +5 -0
  6. RL/Transport50/QC/diverse50/seed_2/complete/SHA256SUMS +27 -0
  7. RL/Transport50/QC/diverse50/seed_2/complete/archive.json +33 -0
  8. RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part000 +3 -0
  9. RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part001 +3 -0
  10. RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part002 +3 -0
  11. RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part003 +3 -0
  12. RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part004 +3 -0
  13. RL/Transport50/QC/diverse50/seed_2/complete/local_archive_verification.json +6 -0
  14. RL/Transport50/QC/diverse50/seed_2/complete/manifest.json +0 -0
  15. RL/Transport50/QC/diverse50/seed_2/complete/publication.json +62 -0
  16. RL/Transport50/QC/diverse50/seed_2/complete/restore.py +38 -0
  17. RL/Transport50/QC/diverse50/seed_2/convergence.json +367 -0
  18. RL/Transport50/QC/diverse50/seed_2/convergence.jsonl +81 -0
  19. RL/Transport50/QC/diverse50/seed_2/eval/final.parquet +3 -0
  20. RL/Transport50/QC/diverse50/seed_2/final/source_metadata.json +314 -0
  21. RL/Transport50/QC/diverse50/seed_2/final/source_resolved_config.json +1127 -0
  22. RL/Transport50/QC/diverse50/seed_2/final/state.pkl +3 -0
  23. RL/Transport50/QC/diverse50/seed_2/frozen_base.json +6 -0
  24. RL/Transport50/QC/diverse50/seed_2/offline_buffer.json +447 -0
  25. RL/Transport50/QC/diverse50/seed_2/resolved_config.json +1127 -0
  26. RL/Transport50/QC/diverse50/seed_2/start_protocol.json +103 -0
  27. RL/Transport50/QC/diverse50/seed_2/status.json +174 -0
  28. RL/Transport50/QC/diverse50/seed_2/training_accounting.json +51 -0
RL/Transport50/QC/diverse50/seed_2/best/source_metadata.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_sha256": "c90217e70689435f766bfc69fa734ffa61175b8cc6c5ee2559faf940490cce49",
3
+ "evaluation": {
4
+ "chunk_transitions": 103000,
5
+ "episodes": 200,
6
+ "eval_index": 41,
7
+ "gain": 0.0010000000000000009,
8
+ "gradient_updates": 309000,
9
+ "ma": 0.537,
10
+ "n_success": 125,
11
+ "policy_version": 103,
12
+ "rl_env_steps": 821308,
13
+ "round": 103,
14
+ "seed_start": 960000,
15
+ "sr": 0.625,
16
+ "success_requested": 0,
17
+ "success_requested_imitation": 0,
18
+ "success_rounds_with_skips": 0,
19
+ "success_skipped": 0,
20
+ "success_skipped_imitation": 0,
21
+ "success_skipped_td": 0,
22
+ "target_step": 820000,
23
+ "update_index": 309000
24
+ },
25
+ "includes": "complete learner state, optimizer, targets, RNG and replay",
26
+ "selection": "observed_periodic_sr"
27
+ }
RL/Transport50/QC/diverse50/seed_2/best/source_resolved_config.json ADDED
@@ -0,0 +1,1127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[0930-S2DEC]",
14
+ "key": "learner.qc.agent.act_bounds_json",
15
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
16
+ "upstream": null,
17
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}"
18
+ },
19
+ {
20
+ "decision_ref": "[0930-S2DEC]",
21
+ "key": "learner.qc.agent.act_max",
22
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
23
+ "upstream": 1.0,
24
+ "value": 4.179931163787842
25
+ },
26
+ {
27
+ "decision_ref": "[0930-S2DEC]",
28
+ "key": "learner.qc.agent.act_min",
29
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
30
+ "upstream": -1.0,
31
+ "value": -3.978297472000122
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.actor_lr_partition",
36
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
37
+ "upstream": "lr=0.0003 shared by actor and critic",
38
+ "value": 0.0001
39
+ },
40
+ {
41
+ "decision_ref": "[1002-S2ALIGN]",
42
+ "key": "learner.qc.agent.actor_num_samples",
43
+ "reason": "single BoN knob including QC TD",
44
+ "upstream": 32,
45
+ "value": 16
46
+ },
47
+ {
48
+ "decision_ref": "[1002-S2ALIGN]",
49
+ "key": "learner.qc.agent.best_of_n",
50
+ "reason": "single BoN knob: collection/eval/QC TD",
51
+ "upstream": 8,
52
+ "value": 16
53
+ },
54
+ {
55
+ "decision_ref": "docs/scope.md [0928-I]",
56
+ "key": "learner.qc.agent.bon_agg",
57
+ "reason": "Best-of-N selection by the ensemble mean",
58
+ "upstream": null,
59
+ "value": "mean"
60
+ },
61
+ {
62
+ "decision_ref": "docs/scope.md [0928-I]",
63
+ "key": "learner.qc.agent.bon_sft",
64
+ "reason": "variant qc flag bon_sft",
65
+ "upstream": "upstream_yaml",
66
+ "value": false
67
+ },
68
+ {
69
+ "decision_ref": "[0930-S2DEC]",
70
+ "key": "learner.qc.agent.critic_lr_partition",
71
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
72
+ "upstream": "lr=0.0003 shared by actor and critic",
73
+ "value": 0.0003
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.partition_optimizer",
78
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
79
+ "upstream": null,
80
+ "value": true
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.q_target_agg",
85
+ "reason": "REDQ-style TD target: min over a random critic subset",
86
+ "upstream": null,
87
+ "value": "subset_min"
88
+ },
89
+ {
90
+ "decision_ref": "docs/scope.md [0928-I]",
91
+ "key": "learner.qc.agent.q_target_replace",
92
+ "reason": "subset drawn without replacement",
93
+ "upstream": null,
94
+ "value": false
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.q_target_subset",
99
+ "reason": "TD-target subset size",
100
+ "upstream": null,
101
+ "value": 2
102
+ },
103
+ {
104
+ "decision_ref": "[0930-S2DEC]",
105
+ "key": "learner.qc.agent.value_hidden_dims",
106
+ "reason": "unified.critic_hidden (critics only; actors native)",
107
+ "upstream": [
108
+ 512,
109
+ 512,
110
+ 512,
111
+ 512
112
+ ],
113
+ "value": [
114
+ 512,
115
+ 512,
116
+ 512
117
+ ]
118
+ },
119
+ {
120
+ "decision_ref": "docs/scope.md [0928-I]",
121
+ "key": "learner.qc.training.bc_pi_steps",
122
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
123
+ "upstream": 1000000,
124
+ "value": 0
125
+ },
126
+ {
127
+ "decision_ref": "docs/scope.md [0928-I]",
128
+ "key": "learner.qc.loop.buffer.utd_online",
129
+ "reason": "UTD=3 per executed action chunk",
130
+ "upstream": 1,
131
+ "value": "3 gradient updates per executed chunk"
132
+ },
133
+ {
134
+ "decision_ref": "docs/scope.md [0928-I]",
135
+ "key": "learner.qc.loop.n_step",
136
+ "reason": "n-step=3 counted in action chunks",
137
+ "upstream": "1 chunk (sequence_length = horizon_length)",
138
+ "value": "3 executed chunks (unified_env_step window)"
139
+ },
140
+ {
141
+ "decision_ref": "docs/scope.md [0928-I]",
142
+ "key": "learner.qc.loop.critic_actor_ratio",
143
+ "reason": "critic every update, actor every 2nd global update",
144
+ "upstream": "1:1 (utd_q = utd_pi)",
145
+ "value": "2:1"
146
+ },
147
+ {
148
+ "decision_ref": "docs/scope.md [0928-I]",
149
+ "key": "learner.qc.loop.buffer.start_training",
150
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
151
+ "upstream": 10000,
152
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
153
+ },
154
+ {
155
+ "decision_ref": "docs/scope.md [0928-I]",
156
+ "key": "learner.qc.loop.buffer.offline_ratio",
157
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
158
+ "upstream": 0.0,
159
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
160
+ },
161
+ {
162
+ "decision_ref": "docs/scope.md [0928-I]",
163
+ "key": "learner.qc.loop.buffer.size",
164
+ "reason": "one shared replay of executed chunks; success buffer = mask",
165
+ "upstream": 2000000,
166
+ "value": "1000000 executed chunks, shared FIFO"
167
+ },
168
+ {
169
+ "decision_ref": "docs/scope.md [0928-I]",
170
+ "key": "learner.qc.actor_architecture",
171
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
172
+ "upstream": [
173
+ 512,
174
+ 512,
175
+ 512,
176
+ 512
177
+ ],
178
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
179
+ },
180
+ {
181
+ "decision_ref": "docs/scope.md [0928-I]",
182
+ "key": "learner.qc.observation",
183
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
184
+ "upstream": "low-dim state",
185
+ "value": "frozen Ri features c [146] from the checkpoint encoder"
186
+ },
187
+ {
188
+ "decision_ref": "[0930-S2DEC]",
189
+ "key": "learner.qc.reward",
190
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
191
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
192
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
193
+ },
194
+ {
195
+ "decision_ref": "[0930-S2DEC]",
196
+ "key": "learner.qc.td_target",
197
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
198
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
199
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
200
+ },
201
+ {
202
+ "decision_ref": "[0930-S2DEC]",
203
+ "key": "learner.qc.eval",
204
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
205
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
206
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
207
+ },
208
+ {
209
+ "decision_ref": "[1002-S2ALIGN]",
210
+ "key": "learner.qc.lr_schedule",
211
+ "reason": "RL env-step clock shared with extra success TD",
212
+ "upstream": "native per-optimizer schedule",
213
+ "value": {
214
+ "actor": 0.0001,
215
+ "critic": 0.0003,
216
+ "decay_end_env_steps": 1606196,
217
+ "min_ratio": 0.1,
218
+ "schedule": "warmup_cosine",
219
+ "temperature": 0.0003,
220
+ "warmup_env_steps": 20000
221
+ }
222
+ },
223
+ {
224
+ "decision_ref": "[0930-S2DEC]",
225
+ "key": "learner.qc.agent.q_min_q_max",
226
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
227
+ "upstream": [
228
+ -999.9999999999991,
229
+ 0.0
230
+ ],
231
+ "value": [
232
+ 0.0,
233
+ 1.0
234
+ ]
235
+ }
236
+ ],
237
+ "protocol": {
238
+ "checkpoint_every_rounds": 5,
239
+ "checkpoint_retention": {
240
+ "best": true,
241
+ "collapse_drop": {
242
+ "min_abs": 0.2,
243
+ "ratio": 0.5
244
+ },
245
+ "diagnostics": true,
246
+ "every_evals": 3
247
+ },
248
+ "collection": {
249
+ "n_envs": 4,
250
+ "train_seed_base": 100000000,
251
+ "train_seed_stride": 1000000,
252
+ "training_freq_chunks": 1000
253
+ },
254
+ "convergence": {
255
+ "early_stop": false,
256
+ "gain_lag": 10,
257
+ "high_sr": 0.95,
258
+ "high_sr_evals": 5,
259
+ "ma_window": 10,
260
+ "max_env_steps": 1606196,
261
+ "min_env_steps": 200000,
262
+ "min_gain": 0.02,
263
+ "zero_success_evals": 8
264
+ },
265
+ "critic_ensemble": {
266
+ "num": 10,
267
+ "policy": "mean",
268
+ "target": {
269
+ "kind": "subset_min",
270
+ "replace": false,
271
+ "size": 2
272
+ }
273
+ },
274
+ "eval": {
275
+ "best_of_n": 16,
276
+ "episodes": 200,
277
+ "every_env_steps": 20000,
278
+ "include_step0": true,
279
+ "seed_start": 960000,
280
+ "workers": 4
281
+ },
282
+ "final_eval": {
283
+ "episodes": 300,
284
+ "on_zero_stop": true,
285
+ "seed_start": 800000
286
+ },
287
+ "n_step_chunks": 3,
288
+ "offline": {
289
+ "adaptive_ratio": false,
290
+ "cohort_size": 50,
291
+ "episodes": 50,
292
+ "learners_without_offline": [
293
+ "dsrl_sac"
294
+ ],
295
+ "ratio": {
296
+ "decay_env_steps": {
297
+ "can": 160000,
298
+ "square": 320000,
299
+ "tool_hang": 640000,
300
+ "transport": 320000
301
+ },
302
+ "end": 0.1,
303
+ "start": {
304
+ "can": 0.7,
305
+ "square": 0.5,
306
+ "tool_hang": 0.5,
307
+ "transport": 0.9
308
+ }
309
+ },
310
+ "selection": "ascending_demo_index",
311
+ "source": "bc_regime",
312
+ "unsupported_learners": [
313
+ "zprl"
314
+ ]
315
+ },
316
+ "replay": {
317
+ "capacity_chunks": 1000000,
318
+ "pin_start_successes": false,
319
+ "success_buffer": "success_mask_on_shared_fifo",
320
+ "success_skip_rule": "skip_below_success_batch_size",
321
+ "unit": "executed_chunk"
322
+ },
323
+ "start": {
324
+ "max_episodes": 200,
325
+ "mode": "offline_buffer",
326
+ "policy": {
327
+ "default": "base_cfm_ode",
328
+ "zprl": "vib_base"
329
+ },
330
+ "target_successes": 20
331
+ },
332
+ "unified": {
333
+ "actor_target_tau": 0.005,
334
+ "critic_hidden": [
335
+ 512,
336
+ 512,
337
+ 512
338
+ ],
339
+ "critic_target_every_critic_step": true,
340
+ "critic_target_tau": 0.005,
341
+ "dsrl_noise_imitation_reference_dims": 28,
342
+ "gamma_env": 0.999,
343
+ "lr": {
344
+ "actor": 0.0001,
345
+ "critic": 0.0003,
346
+ "decay_end_env_steps": 1606196,
347
+ "min_ratio": 0.1,
348
+ "schedule": "warmup_cosine",
349
+ "temperature": 0.0003,
350
+ "warmup_env_steps": 20000
351
+ },
352
+ "reward": "binary_first_success",
353
+ "success_td_polyak": true,
354
+ "timeout_bootstrap": false,
355
+ "zprl_imitation": "mse_pretanh_mean"
356
+ },
357
+ "updates": {
358
+ "batch_size": 256,
359
+ "critic_per_actor": 2,
360
+ "success_batch_size": 256,
361
+ "utd_per_chunk": 3
362
+ },
363
+ "version": "study2_protocol_v10"
364
+ },
365
+ "protocol_hash": "bdfdfb18fc67ba88af67e02b6592b190b509c5ad68cbd7357fcbca9fa8c796b9",
366
+ "provenance_extra": {
367
+ "agent_library": {
368
+ "framework": "jax",
369
+ "sources": {
370
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
371
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
372
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
373
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
374
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
375
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
376
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
377
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
378
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
379
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
380
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
381
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
382
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
383
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
384
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
385
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
386
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
387
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
388
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
389
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
390
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
391
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
392
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
393
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
394
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
395
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
396
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
397
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
398
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
399
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
400
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
401
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
402
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
403
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
404
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
405
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
406
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
407
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
408
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
409
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
410
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
411
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
412
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
413
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
414
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
415
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
416
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
417
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
418
+ "train.py": "d1ec607894c8e126e710b5a4ef6c48a4e9bf6c66c0872ce2e2bdc1c65b66d98d",
419
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
420
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
421
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
422
+ },
423
+ "version": "study2_jax_agents_v2"
424
+ },
425
+ "base_framework": "jax",
426
+ "bc_identity": {
427
+ "architecture": "dice_vision_resnet_flow_unet_v1",
428
+ "family": "StandardBaseBC",
429
+ "framework": "jax"
430
+ },
431
+ "checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261022_transport_focused50_v10_four/base/diverse50/epoch_3000.pkl",
432
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
433
+ "experiment": "transport_8",
434
+ "frozen_hash": "2414629d8dcbea1d75ca2831d5e0b649e65cffd56aa0ed22212fd0c3ec60b1bc",
435
+ "learner": {
436
+ "action_bounds": {
437
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
438
+ "compat_act_max": 4.179931163787842,
439
+ "compat_act_min": -3.978297472000122,
440
+ "hi": [
441
+ 1.0,
442
+ 1.0,
443
+ 1.0,
444
+ 1.0,
445
+ 2.775709390640259,
446
+ 2.2864010334014893,
447
+ 1.0,
448
+ 1.0,
449
+ 1.0,
450
+ 1.0,
451
+ 1.4753823280334473,
452
+ 4.088724613189697,
453
+ 4.179931163787842,
454
+ 1.0
455
+ ],
456
+ "kind": "per_dimension",
457
+ "lo": [
458
+ -1.0,
459
+ -1.0,
460
+ -1.0,
461
+ -1.4882197380065918,
462
+ -3.233746290206909,
463
+ -2.5405781269073486,
464
+ -1.0,
465
+ -1.0,
466
+ -1.0,
467
+ -1.0,
468
+ -1.7870254516601562,
469
+ -3.208747148513794,
470
+ -3.978297472000122,
471
+ -1.0
472
+ ],
473
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
474
+ },
475
+ "agent_config": {
476
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
477
+ "act_max": 4.179931163787842,
478
+ "act_min": -3.978297472000122,
479
+ "action_chunking": true,
480
+ "action_critic_hidden_dims": null,
481
+ "action_dim": 14,
482
+ "action_num_qs": 10,
483
+ "action_repeat": false,
484
+ "actor_backbone": "mlp",
485
+ "actor_decay_steps": 50000,
486
+ "actor_end_value": 2e-05,
487
+ "actor_hidden_dims": [
488
+ 512,
489
+ 512,
490
+ 512,
491
+ 512
492
+ ],
493
+ "actor_layer_norm": false,
494
+ "actor_lr": 0.0003,
495
+ "actor_lr_partition": 0.0001,
496
+ "actor_num_samples": 16,
497
+ "actor_obs": "state",
498
+ "actor_scheduler": "cosine",
499
+ "actor_tau": 0.005,
500
+ "actor_type": "best-of-n",
501
+ "actor_warmup_steps": 2000,
502
+ "actor_weight_decay": 0.0,
503
+ "adv_clip_min": null,
504
+ "agent_name": "acfql",
505
+ "alpha": 500,
506
+ "batch_size": 256,
507
+ "best_of_n": 16,
508
+ "bon_agg": "mean",
509
+ "bon_sft": false,
510
+ "bon_sft_coeff": 1.0,
511
+ "bootstrap_ratio": 0.5,
512
+ "bptt_num_samples": 4,
513
+ "calql_q_subsample": 2,
514
+ "clip_grad_norm": 1000.0,
515
+ "clip_intermediate_actions": true,
516
+ "clip_max_epsilon_multiplier": 1.0,
517
+ "clip_min_epsilon_multiplier": 1.0,
518
+ "constant_noise_std": 0.01,
519
+ "cql_alpha": 5.0,
520
+ "cql_n_actions": 10,
521
+ "cql_temp": 1.0,
522
+ "critic_backbone": "mlp",
523
+ "critic_decay_steps": 5000,
524
+ "critic_end_value": 1e-08,
525
+ "critic_loss_type": "mse",
526
+ "critic_lr": 0.0003,
527
+ "critic_lr_partition": 0.0003,
528
+ "critic_obs": "state",
529
+ "critic_scheduler": "constant",
530
+ "critic_warmup_steps": 500,
531
+ "critic_weight_decay": 1e-05,
532
+ "denoised_clip_value": 1.0,
533
+ "discount": 0.999,
534
+ "edit_action_scale": 0.5,
535
+ "edit_actor_hidden_dims": null,
536
+ "edit_hidden_dims": [
537
+ 256,
538
+ 256,
539
+ 256
540
+ ],
541
+ "edit_init_temperature": 1.0,
542
+ "edit_layer_norm": true,
543
+ "edit_lr": 0.0003,
544
+ "edit_policy_bound": 0.1,
545
+ "edit_start_step": 100000,
546
+ "edit_target_entropy": null,
547
+ "edit_weight_decay": 0.0,
548
+ "encoder": null,
549
+ "entropy_backup": false,
550
+ "flow_steps": 10,
551
+ "fql_distillation_coeff": 10.0,
552
+ "fql_train_main_policy": true,
553
+ "freeze_vision_encoder": false,
554
+ "ft_flow_steps": 10,
555
+ "horizon_length": 8,
556
+ "init_temperature": 1.0,
557
+ "layer_norm": true,
558
+ "lr": 0.0003,
559
+ "lr_action_critic": null,
560
+ "lr_actor": null,
561
+ "lr_actor_bc_flow": null,
562
+ "lr_critic": null,
563
+ "lr_edit_actor": null,
564
+ "lr_noise_critic": null,
565
+ "max_noise_std": 0.01,
566
+ "mc_regression": false,
567
+ "mc_regression_coeff": 1.0,
568
+ "min_noise_std": 0.01,
569
+ "mip_q_ensemble": false,
570
+ "mip_q_noise_scale": 1.0,
571
+ "mip_q_t_star": 0.9,
572
+ "mip_t_star": 0.999,
573
+ "muon_beta": 0.95,
574
+ "muon_nesterov": true,
575
+ "muon_ns_steps": 5,
576
+ "n_action_samples": 8,
577
+ "n_edit_samples": 8,
578
+ "noise_critic_grad_steps": 1,
579
+ "noise_dim": null,
580
+ "noise_net_hidden_dims": [
581
+ 256,
582
+ 256
583
+ ],
584
+ "noise_net_layer_norm": true,
585
+ "normalize_act_space_dimension": false,
586
+ "normalize_denoising_horizon": false,
587
+ "normalize_q_loss": false,
588
+ "num_bins": 256,
589
+ "num_ensemble_members": 10,
590
+ "num_qs": 10,
591
+ "ob_dims": [
592
+ 146
593
+ ],
594
+ "one_step_decay_steps": null,
595
+ "one_step_end_value": null,
596
+ "one_step_hidden_dims": [
597
+ 512,
598
+ 512,
599
+ 512,
600
+ 512
601
+ ],
602
+ "one_step_layer_norm": false,
603
+ "one_step_lr": null,
604
+ "one_step_scheduler": null,
605
+ "one_step_warmup_steps": null,
606
+ "optimizer_action_critic": null,
607
+ "optimizer_actor": "adam",
608
+ "optimizer_actor_bc_flow": "adam",
609
+ "optimizer_critic": "adam",
610
+ "optimizer_edit_actor": null,
611
+ "paligemma_tokenizer_path": null,
612
+ "partition_optimizer": true,
613
+ "policy_type": "flow",
614
+ "ppo_batch_size": 256,
615
+ "ppo_lr": 4.5e-05,
616
+ "proprio_dim": null,
617
+ "q_agg": "mean",
618
+ "q_max": 1.0,
619
+ "q_min": 0.0,
620
+ "q_target_agg": "subset_min",
621
+ "q_target_replace": false,
622
+ "q_target_subset": 2,
623
+ "q_variance_reduction": false,
624
+ "q_vr_num_samples": 8,
625
+ "q_vr_reduction": "mean",
626
+ "randn_clip_value": 3.0,
627
+ "shortcut_denoising_steps": 4,
628
+ "shortcut_inference_steps": 4,
629
+ "simba_actor_hidden_dims": [
630
+ 1024,
631
+ 1024
632
+ ],
633
+ "simba_critic_hidden_dims": [
634
+ 1024,
635
+ 1024,
636
+ 1024
637
+ ],
638
+ "subsample_bon": false,
639
+ "target_entropy": null,
640
+ "tau": 0.005,
641
+ "tf_pi_conditioning": "adaln",
642
+ "tf_pi_dropout": 0.0,
643
+ "tf_pi_embed_dim": 256,
644
+ "tf_pi_heads": 4,
645
+ "tf_pi_layers": 8,
646
+ "tf_q_conditioning": "adaln",
647
+ "tf_q_dropout": 0.0,
648
+ "tf_q_embed_dim": 256,
649
+ "tf_q_heads": 4,
650
+ "tf_q_layers": 8,
651
+ "time_embedding": "sinusoidal",
652
+ "time_embedding_dim": 32,
653
+ "train_one_step_in_offline": true,
654
+ "train_one_step_in_online": true,
655
+ "useFiLM": false,
656
+ "useSimBa": false,
657
+ "useTF": false,
658
+ "use_constant_noise": true,
659
+ "use_constant_scheduler_for_bc": false,
660
+ "use_denoiser": false,
661
+ "use_edit_actor": false,
662
+ "use_mip_q": false,
663
+ "use_muon": false,
664
+ "use_one_step_policy": false,
665
+ "use_shortcut": false,
666
+ "use_state": "proprio",
667
+ "use_success_buffer_q": false,
668
+ "value_hidden_dims": [
669
+ 512,
670
+ 512,
671
+ 512
672
+ ],
673
+ "vit_encoder": "minvit",
674
+ "vit_img_proj_dim": 0,
675
+ "vit_state_proj_dim": 0
676
+ },
677
+ "convention": "unified_env_step",
678
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
679
+ "gamma_env": 0.999,
680
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
681
+ "learner": "qc",
682
+ "optimizers": {
683
+ "actor_lr": 0.0001,
684
+ "critic_lr": 0.0003,
685
+ "reset_optimizers_with_lr": "not used",
686
+ "schedule": {
687
+ "actor": 0.0001,
688
+ "critic": 0.0003,
689
+ "decay_end_env_steps": 1606196,
690
+ "min_ratio": 0.1,
691
+ "schedule": "warmup_cosine",
692
+ "temperature": 0.0003,
693
+ "warmup_env_steps": 20000
694
+ }
695
+ },
696
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
697
+ "recipe": {
698
+ "derived": {
699
+ "agent.actor_num_samples": {
700
+ "after": 8,
701
+ "before": 32
702
+ },
703
+ "agent.actor_type": {
704
+ "after": "best-of-n",
705
+ "before": "upstream_yaml"
706
+ },
707
+ "agent.bon_sft": {
708
+ "after": false,
709
+ "before": "upstream_yaml"
710
+ }
711
+ },
712
+ "explicit_overrides": {},
713
+ "source": "third_party/ogpo/scripts/qc/transport.sh",
714
+ "upstream_exact": false
715
+ },
716
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
717
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
718
+ "scan_block": 100,
719
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
720
+ "success_terms": {
721
+ "bc_on_success_batch": false,
722
+ "critic_success_td": false,
723
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
724
+ },
725
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
726
+ "unified_notes": {
727
+ "critic_target_every_critic_step": {
728
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
729
+ "value": true
730
+ },
731
+ "success_td_polyak": {
732
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
733
+ "value": true
734
+ }
735
+ },
736
+ "upstream_config": {
737
+ "agent": {
738
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
739
+ "act_max": 4.179931163787842,
740
+ "act_min": -3.978297472000122,
741
+ "action_chunking": true,
742
+ "action_critic_hidden_dims": null,
743
+ "action_dim": null,
744
+ "action_num_qs": 10,
745
+ "action_repeat": false,
746
+ "actor_backbone": "mlp",
747
+ "actor_decay_steps": 50000,
748
+ "actor_end_value": 2e-05,
749
+ "actor_hidden_dims": [
750
+ 512,
751
+ 512,
752
+ 512,
753
+ 512
754
+ ],
755
+ "actor_layer_norm": false,
756
+ "actor_lr": 0.0003,
757
+ "actor_lr_partition": 0.0001,
758
+ "actor_num_samples": 16,
759
+ "actor_obs": "state",
760
+ "actor_scheduler": "cosine",
761
+ "actor_tau": 0.005,
762
+ "actor_type": "best-of-n",
763
+ "actor_warmup_steps": 2000,
764
+ "actor_weight_decay": 0.0,
765
+ "adv_clip_min": null,
766
+ "agent_name": "acfql",
767
+ "alpha": 500,
768
+ "batch_size": 256,
769
+ "best_of_n": 16,
770
+ "bon_agg": "mean",
771
+ "bon_sft": false,
772
+ "bon_sft_coeff": 1.0,
773
+ "bootstrap_ratio": 0.5,
774
+ "bptt_num_samples": 4,
775
+ "calql_q_subsample": 2,
776
+ "clip_grad_norm": 1000.0,
777
+ "clip_intermediate_actions": true,
778
+ "clip_max_epsilon_multiplier": 1.0,
779
+ "clip_min_epsilon_multiplier": 1.0,
780
+ "constant_noise_std": 0.01,
781
+ "cql_alpha": 5.0,
782
+ "cql_n_actions": 10,
783
+ "cql_temp": 1.0,
784
+ "critic_backbone": "mlp",
785
+ "critic_decay_steps": 5000,
786
+ "critic_end_value": 1e-08,
787
+ "critic_loss_type": "mse",
788
+ "critic_lr": 0.0003,
789
+ "critic_lr_partition": 0.0003,
790
+ "critic_obs": "state",
791
+ "critic_scheduler": "constant",
792
+ "critic_warmup_steps": 500,
793
+ "critic_weight_decay": 1e-05,
794
+ "denoised_clip_value": 1.0,
795
+ "discount": 0.999,
796
+ "edit_action_scale": 0.5,
797
+ "edit_actor_hidden_dims": null,
798
+ "edit_hidden_dims": [
799
+ 256,
800
+ 256,
801
+ 256
802
+ ],
803
+ "edit_init_temperature": 1.0,
804
+ "edit_layer_norm": true,
805
+ "edit_lr": 0.0003,
806
+ "edit_policy_bound": 0.1,
807
+ "edit_start_step": 100000,
808
+ "edit_target_entropy": null,
809
+ "edit_weight_decay": 0.0,
810
+ "encoder": null,
811
+ "entropy_backup": false,
812
+ "flow_steps": 10,
813
+ "fql_distillation_coeff": 10.0,
814
+ "fql_train_main_policy": true,
815
+ "freeze_vision_encoder": false,
816
+ "ft_flow_steps": 10,
817
+ "horizon_length": 8,
818
+ "init_temperature": 1.0,
819
+ "layer_norm": true,
820
+ "lr": 0.0003,
821
+ "lr_action_critic": null,
822
+ "lr_actor": null,
823
+ "lr_actor_bc_flow": null,
824
+ "lr_critic": null,
825
+ "lr_edit_actor": null,
826
+ "lr_noise_critic": null,
827
+ "max_noise_std": 0.01,
828
+ "mc_regression": false,
829
+ "mc_regression_coeff": 1.0,
830
+ "min_noise_std": 0.01,
831
+ "mip_q_ensemble": false,
832
+ "mip_q_noise_scale": 1.0,
833
+ "mip_q_t_star": 0.9,
834
+ "mip_t_star": 0.999,
835
+ "muon_beta": 0.95,
836
+ "muon_nesterov": true,
837
+ "muon_ns_steps": 5,
838
+ "n_action_samples": 8,
839
+ "n_edit_samples": 8,
840
+ "noise_critic_grad_steps": 1,
841
+ "noise_dim": null,
842
+ "noise_net_hidden_dims": [
843
+ 256,
844
+ 256
845
+ ],
846
+ "noise_net_layer_norm": true,
847
+ "normalize_act_space_dimension": false,
848
+ "normalize_denoising_horizon": false,
849
+ "normalize_q_loss": false,
850
+ "num_bins": 256,
851
+ "num_ensemble_members": 10,
852
+ "num_qs": 10,
853
+ "ob_dims": null,
854
+ "one_step_decay_steps": null,
855
+ "one_step_end_value": null,
856
+ "one_step_hidden_dims": [
857
+ 512,
858
+ 512,
859
+ 512,
860
+ 512
861
+ ],
862
+ "one_step_layer_norm": false,
863
+ "one_step_lr": null,
864
+ "one_step_scheduler": null,
865
+ "one_step_warmup_steps": null,
866
+ "optimizer_action_critic": null,
867
+ "optimizer_actor": "adam",
868
+ "optimizer_actor_bc_flow": "adam",
869
+ "optimizer_critic": "adam",
870
+ "optimizer_edit_actor": null,
871
+ "paligemma_tokenizer_path": null,
872
+ "partition_optimizer": true,
873
+ "policy_type": "flow",
874
+ "ppo_batch_size": 256,
875
+ "ppo_lr": 4.5e-05,
876
+ "proprio_dim": null,
877
+ "q_agg": "mean",
878
+ "q_max": null,
879
+ "q_min": null,
880
+ "q_target_agg": "subset_min",
881
+ "q_target_replace": false,
882
+ "q_target_subset": 2,
883
+ "q_variance_reduction": false,
884
+ "q_vr_num_samples": 8,
885
+ "q_vr_reduction": "mean",
886
+ "randn_clip_value": 3.0,
887
+ "shortcut_denoising_steps": 4,
888
+ "shortcut_inference_steps": 4,
889
+ "simba_actor_hidden_dims": [
890
+ 1024,
891
+ 1024
892
+ ],
893
+ "simba_critic_hidden_dims": [
894
+ 1024,
895
+ 1024,
896
+ 1024
897
+ ],
898
+ "subsample_bon": false,
899
+ "target_entropy": null,
900
+ "tau": 0.005,
901
+ "tf_pi_conditioning": "adaln",
902
+ "tf_pi_dropout": 0.0,
903
+ "tf_pi_embed_dim": 256,
904
+ "tf_pi_heads": 4,
905
+ "tf_pi_layers": 8,
906
+ "tf_q_conditioning": "adaln",
907
+ "tf_q_dropout": 0.0,
908
+ "tf_q_embed_dim": 256,
909
+ "tf_q_heads": 4,
910
+ "tf_q_layers": 8,
911
+ "time_embedding": "sinusoidal",
912
+ "time_embedding_dim": 32,
913
+ "train_one_step_in_offline": true,
914
+ "train_one_step_in_online": true,
915
+ "useFiLM": false,
916
+ "useSimBa": false,
917
+ "useTF": false,
918
+ "use_constant_noise": true,
919
+ "use_constant_scheduler_for_bc": false,
920
+ "use_denoiser": false,
921
+ "use_edit_actor": false,
922
+ "use_mip_q": false,
923
+ "use_muon": false,
924
+ "use_one_step_policy": false,
925
+ "use_shortcut": false,
926
+ "use_state": "proprio",
927
+ "use_success_buffer_q": false,
928
+ "value_hidden_dims": [
929
+ 512,
930
+ 512,
931
+ 512
932
+ ],
933
+ "vit_encoder": "minvit",
934
+ "vit_img_proj_dim": 0,
935
+ "vit_state_proj_dim": 0
936
+ },
937
+ "bc": {
938
+ "clip_bc": false,
939
+ "clip_bc_threshold": 0.45,
940
+ "clip_bc_wrt": "ODE"
941
+ },
942
+ "buffer": {
943
+ "offline_ratio": 0.0,
944
+ "size": 2000000,
945
+ "start_training": 10000,
946
+ "utd_online": 1,
947
+ "utd_pi": null,
948
+ "utd_q": null,
949
+ "utd_v": null,
950
+ "utd_warmup": 1
951
+ },
952
+ "checkpoint": {
953
+ "ep_resume": 0,
954
+ "restore_actor_path": null,
955
+ "restore_critic_path": null,
956
+ "restore_encoder_path": null
957
+ },
958
+ "dataset": {
959
+ "num_offline_trajs": -1,
960
+ "ogbench_dir": null,
961
+ "p_aug": 0.0,
962
+ "proportion": 1.0,
963
+ "replace_interval": 1000
964
+ },
965
+ "env": {
966
+ "name": "transport-mh-low_dim",
967
+ "post_success_steps": 0,
968
+ "rew_fn": "sparse",
969
+ "sparse": false
970
+ },
971
+ "eval": {
972
+ "eval_episodes": 2,
973
+ "n_eval_envs": 32,
974
+ "plot_q_vs_mc": false,
975
+ "video_episodes": 0,
976
+ "video_frame_skip": 3
977
+ },
978
+ "experiment": {
979
+ "log": true,
980
+ "project": "QC",
981
+ "run_group": "transport",
982
+ "run_name": "qc_transport",
983
+ "save_dir": "exp/ogpo",
984
+ "seed": 1,
985
+ "wandb_name": "qc_transport"
986
+ },
987
+ "intervals": {
988
+ "eval_interval": 20000,
989
+ "eval_interval_bc": 5000,
990
+ "log_interval": 5000,
991
+ "preemption_backup_interval": -1,
992
+ "save_interval": 500000,
993
+ "save_interval_bc": -1,
994
+ "step_data_log_interval": -1
995
+ },
996
+ "success_buffer": {
997
+ "batch_size": 256,
998
+ "use_success_buffer": false
999
+ },
1000
+ "training": {
1001
+ "bc_only": false,
1002
+ "bc_pi_steps": 0,
1003
+ "bc_q_steps": 0,
1004
+ "bc_refine_steps": 0,
1005
+ "calql_steps": 0,
1006
+ "force_jax_sync": true,
1007
+ "gpu_mode": "single",
1008
+ "online_steps": 6000000,
1009
+ "q_warmup_steps": 0
1010
+ }
1011
+ },
1012
+ "upstream_source": {
1013
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1014
+ "patches_sha256": {
1015
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1016
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1017
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1018
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1019
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1020
+ }
1021
+ }
1022
+ },
1023
+ "observation_mode": "pixel_state",
1024
+ "offline_buffer": {
1025
+ "actions_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1026
+ "aux_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1027
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
1028
+ "chunks": 3183,
1029
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1030
+ "decision_ref": "[1017-S2OFFBC]",
1031
+ "env_steps": 25283,
1032
+ "features_sha256": "9f82519c4cb371ffbbb2c3f5ae9208cb7dc6e48ca9e0ba28b7c3845dc1f1148f",
1033
+ "horizon": 8,
1034
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
1035
+ "learner_coordinates": "JaxStudy2Learner:qc",
1036
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1037
+ "ratio": {
1038
+ "adaptive_ratio": false,
1039
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1040
+ },
1041
+ "selection": {
1042
+ "cohort_size": 50,
1043
+ "demos": [
1044
+ "demo_0",
1045
+ "demo_7",
1046
+ "demo_8",
1047
+ "demo_10",
1048
+ "demo_11",
1049
+ "demo_13",
1050
+ "demo_14",
1051
+ "demo_17",
1052
+ "demo_20",
1053
+ "demo_22",
1054
+ "demo_28",
1055
+ "demo_29",
1056
+ "demo_33",
1057
+ "demo_38",
1058
+ "demo_39",
1059
+ "demo_48",
1060
+ "demo_49",
1061
+ "demo_53",
1062
+ "demo_59",
1063
+ "demo_61",
1064
+ "demo_69",
1065
+ "demo_75",
1066
+ "demo_76",
1067
+ "demo_81",
1068
+ "demo_83",
1069
+ "demo_88",
1070
+ "demo_95",
1071
+ "demo_96",
1072
+ "demo_102",
1073
+ "demo_103",
1074
+ "demo_105",
1075
+ "demo_108",
1076
+ "demo_119",
1077
+ "demo_120",
1078
+ "demo_126",
1079
+ "demo_128",
1080
+ "demo_139",
1081
+ "demo_151",
1082
+ "demo_152",
1083
+ "demo_154",
1084
+ "demo_159",
1085
+ "demo_163",
1086
+ "demo_165",
1087
+ "demo_171",
1088
+ "demo_173",
1089
+ "demo_177",
1090
+ "demo_180",
1091
+ "demo_181",
1092
+ "demo_182",
1093
+ "demo_190"
1094
+ ],
1095
+ "manifest": "/ssd/WorkSpace/RL-Postrain/data/bc/versions/ph_distribution_v2/manifests/transport.json",
1096
+ "manifest_sha256": "8f2569c5918df85bccd661f8b4c8f76fd73948ca0c4ff96487e949ddbdb4013f",
1097
+ "regime": "diverse50",
1098
+ "source": "bc_regime",
1099
+ "source_ph_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1100
+ "split": {
1101
+ "training": 45,
1102
+ "validation": 5
1103
+ },
1104
+ "task": "transport"
1105
+ }
1106
+ },
1107
+ "offline_role": "mixed",
1108
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1109
+ "smoke": false,
1110
+ "task": "transport",
1111
+ "task_spec_hash": "fd9c198a613abab85d95bda9521355ae6d926bcf7da1895ca7a1dd03325117cb"
1112
+ },
1113
+ "seed": 2,
1114
+ "start_policy": "base_cfm_ode",
1115
+ "variant": {
1116
+ "flags": {
1117
+ "bon_sft": false,
1118
+ "use_success_buffer": false
1119
+ },
1120
+ "learner": "qc",
1121
+ "name": "qc",
1122
+ "phase": 1,
1123
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1124
+ "version": "study2_variants_v4"
1125
+ },
1126
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1127
+ }
RL/Transport50/QC/diverse50/seed_2/best/state.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c90217e70689435f766bfc69fa734ffa61175b8cc6c5ee2559faf940490cce49
3
+ size 214992763
RL/Transport50/QC/diverse50/seed_2/comparison_evaluations.jsonl ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"eval_index": 0, "rl_env_steps": 0, "target_step": 0, "sr": 0.505, "round": 0, "update_index": 0, "policy_version": 0, "chunk_transitions": 0, "gradient_updates": 0, "n_success": 101, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
2
+ {"eval_index": 1, "rl_env_steps": 23934, "target_step": 20000, "sr": 0.46, "round": 3, "update_index": 9000, "policy_version": 3, "chunk_transitions": 3000, "gradient_updates": 9000, "n_success": 92, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
3
+ {"eval_index": 2, "rl_env_steps": 47852, "target_step": 40000, "sr": 0.49, "round": 6, "update_index": 18000, "policy_version": 6, "chunk_transitions": 6000, "gradient_updates": 18000, "n_success": 98, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
4
+ {"eval_index": 3, "rl_env_steps": 63816, "target_step": 60000, "sr": 0.41, "round": 8, "update_index": 24000, "policy_version": 8, "chunk_transitions": 8000, "gradient_updates": 24000, "n_success": 82, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
5
+ {"eval_index": 4, "rl_env_steps": 87736, "target_step": 80000, "sr": 0.535, "round": 11, "update_index": 33000, "policy_version": 11, "chunk_transitions": 11000, "gradient_updates": 33000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
6
+ {"eval_index": 5, "rl_env_steps": 103698, "target_step": 100000, "sr": 0.495, "round": 13, "update_index": 39000, "policy_version": 13, "chunk_transitions": 13000, "gradient_updates": 39000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
7
+ {"eval_index": 6, "rl_env_steps": 127620, "target_step": 120000, "sr": 0.535, "round": 16, "update_index": 48000, "policy_version": 16, "chunk_transitions": 16000, "gradient_updates": 48000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
8
+ {"eval_index": 7, "rl_env_steps": 143590, "target_step": 140000, "sr": 0.52, "round": 18, "update_index": 54000, "policy_version": 18, "chunk_transitions": 18000, "gradient_updates": 54000, "n_success": 104, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
9
+ {"eval_index": 8, "rl_env_steps": 167527, "target_step": 160000, "sr": 0.51, "round": 21, "update_index": 63000, "policy_version": 21, "chunk_transitions": 21000, "gradient_updates": 63000, "n_success": 102, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
10
+ {"eval_index": 9, "rl_env_steps": 183481, "target_step": 180000, "sr": 0.565, "round": 23, "update_index": 69000, "policy_version": 23, "chunk_transitions": 23000, "gradient_updates": 69000, "n_success": 113, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5025000000000001, "gain": null, "observed": true, "imputed": false}
11
+ {"eval_index": 10, "rl_env_steps": 207405, "target_step": 200000, "sr": 0.495, "round": 26, "update_index": 78000, "policy_version": 26, "chunk_transitions": 26000, "gradient_updates": 78000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5015, "gain": null, "observed": true, "imputed": false}
12
+ {"eval_index": 11, "rl_env_steps": 223339, "target_step": 220000, "sr": 0.51, "round": 28, "update_index": 84000, "policy_version": 28, "chunk_transitions": 28000, "gradient_updates": 84000, "n_success": 102, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5065, "gain": null, "observed": true, "imputed": false}
13
+ {"eval_index": 12, "rl_env_steps": 247250, "target_step": 240000, "sr": 0.45, "round": 31, "update_index": 93000, "policy_version": 31, "chunk_transitions": 31000, "gradient_updates": 93000, "n_success": 90, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5025, "gain": null, "observed": true, "imputed": false}
14
+ {"eval_index": 13, "rl_env_steps": 263200, "target_step": 260000, "sr": 0.465, "round": 33, "update_index": 99000, "policy_version": 33, "chunk_transitions": 33000, "gradient_updates": 99000, "n_success": 93, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.508, "gain": null, "observed": true, "imputed": false}
15
+ {"eval_index": 14, "rl_env_steps": 287120, "target_step": 280000, "sr": 0.495, "round": 36, "update_index": 108000, "policy_version": 36, "chunk_transitions": 36000, "gradient_updates": 108000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.504, "gain": null, "observed": true, "imputed": false}
16
+ {"eval_index": 15, "rl_env_steps": 303073, "target_step": 300000, "sr": 0.495, "round": 38, "update_index": 114000, "policy_version": 38, "chunk_transitions": 38000, "gradient_updates": 114000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.504, "gain": null, "observed": true, "imputed": false}
17
+ {"eval_index": 16, "rl_env_steps": 327018, "target_step": 320000, "sr": 0.46, "round": 41, "update_index": 123000, "policy_version": 41, "chunk_transitions": 41000, "gradient_updates": 123000, "n_success": 92, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.4965, "gain": null, "observed": true, "imputed": false}
18
+ {"eval_index": 17, "rl_env_steps": 342983, "target_step": 340000, "sr": 0.53, "round": 43, "update_index": 129000, "policy_version": 43, "chunk_transitions": 43000, "gradient_updates": 129000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.49750000000000005, "gain": null, "observed": true, "imputed": false}
19
+ {"eval_index": 18, "rl_env_steps": 366917, "target_step": 360000, "sr": 0.545, "round": 46, "update_index": 138000, "policy_version": 46, "chunk_transitions": 46000, "gradient_updates": 138000, "n_success": 109, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.501, "gain": null, "observed": true, "imputed": false}
20
+ {"eval_index": 19, "rl_env_steps": 382870, "target_step": 380000, "sr": 0.535, "round": 48, "update_index": 144000, "policy_version": 48, "chunk_transitions": 48000, "gradient_updates": 144000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.49800000000000005, "gain": -0.004500000000000004, "observed": true, "imputed": false}
21
+ {"eval_index": 20, "rl_env_steps": 406795, "target_step": 400000, "sr": 0.59, "round": 51, "update_index": 153000, "policy_version": 51, "chunk_transitions": 51000, "gradient_updates": 153000, "n_success": 118, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5075000000000001, "gain": 0.006000000000000116, "observed": true, "imputed": false}
22
+ {"eval_index": 21, "rl_env_steps": 422759, "target_step": 420000, "sr": 0.48, "round": 53, "update_index": 159000, "policy_version": 53, "chunk_transitions": 53000, "gradient_updates": 159000, "n_success": 96, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5045, "gain": -0.0020000000000000018, "observed": true, "imputed": false}
23
+ {"eval_index": 22, "rl_env_steps": 446688, "target_step": 440000, "sr": 0.435, "round": 56, "update_index": 168000, "policy_version": 56, "chunk_transitions": 56000, "gradient_updates": 168000, "n_success": 87, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.503, "gain": 0.000500000000000056, "observed": true, "imputed": false}
24
+ {"eval_index": 23, "rl_env_steps": 462657, "target_step": 460000, "sr": 0.53, "round": 58, "update_index": 174000, "policy_version": 58, "chunk_transitions": 58000, "gradient_updates": 174000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5095, "gain": 0.0014999999999999458, "observed": true, "imputed": false}
25
+ {"eval_index": 24, "rl_env_steps": 486562, "target_step": 480000, "sr": 0.555, "round": 61, "update_index": 183000, "policy_version": 61, "chunk_transitions": 61000, "gradient_updates": 183000, "n_success": 111, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5155000000000001, "gain": 0.011500000000000066, "observed": true, "imputed": false}
26
+ {"eval_index": 25, "rl_env_steps": 502524, "target_step": 500000, "sr": 0.585, "round": 63, "update_index": 189000, "policy_version": 63, "chunk_transitions": 63000, "gradient_updates": 189000, "n_success": 117, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5245, "gain": 0.020499999999999963, "observed": true, "imputed": false}
27
+ {"eval_index": 26, "rl_env_steps": 526433, "target_step": 520000, "sr": 0.565, "round": 66, "update_index": 198000, "policy_version": 66, "chunk_transitions": 66000, "gradient_updates": 198000, "n_success": 113, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5349999999999999, "gain": 0.03849999999999992, "observed": true, "imputed": false}
28
+ {"eval_index": 27, "rl_env_steps": 542357, "target_step": 540000, "sr": 0.55, "round": 68, "update_index": 204000, "policy_version": 68, "chunk_transitions": 68000, "gradient_updates": 204000, "n_success": 110, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.537, "gain": 0.03949999999999998, "observed": true, "imputed": false}
29
+ {"eval_index": 28, "rl_env_steps": 566281, "target_step": 560000, "sr": 0.575, "round": 71, "update_index": 213000, "policy_version": 71, "chunk_transitions": 71000, "gradient_updates": 213000, "n_success": 115, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.54, "gain": 0.039000000000000035, "observed": true, "imputed": false}
30
+ {"eval_index": 29, "rl_env_steps": 582252, "target_step": 580000, "sr": 0.53, "round": 73, "update_index": 219000, "policy_version": 73, "chunk_transitions": 73000, "gradient_updates": 219000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5395000000000001, "gain": 0.04150000000000004, "observed": true, "imputed": false}
31
+ {"eval_index": 30, "rl_env_steps": 606153, "target_step": 600000, "sr": 0.495, "round": 76, "update_index": 228000, "policy_version": 76, "chunk_transitions": 76000, "gradient_updates": 228000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.53, "gain": 0.022499999999999964, "observed": true, "imputed": false}
32
+ {"eval_index": 31, "rl_env_steps": 622092, "target_step": 620000, "sr": 0.54, "round": 78, "update_index": 234000, "policy_version": 78, "chunk_transitions": 78000, "gradient_updates": 234000, "n_success": 108, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.536, "gain": 0.031500000000000083, "observed": true, "imputed": false}
33
+ {"eval_index": 32, "rl_env_steps": 645964, "target_step": 640000, "sr": 0.49, "round": 81, "update_index": 243000, "policy_version": 81, "chunk_transitions": 81000, "gradient_updates": 243000, "n_success": 98, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5415, "gain": 0.03849999999999998, "observed": true, "imputed": false}
34
+ {"eval_index": 33, "rl_env_steps": 661926, "target_step": 660000, "sr": 0.535, "round": 83, "update_index": 249000, "policy_version": 83, "chunk_transitions": 83000, "gradient_updates": 249000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.542, "gain": 0.032500000000000084, "observed": true, "imputed": false}
35
+ {"eval_index": 34, "rl_env_steps": 685829, "target_step": 680000, "sr": 0.56, "round": 86, "update_index": 258000, "policy_version": 86, "chunk_transitions": 86000, "gradient_updates": 258000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5425000000000001, "gain": 0.027000000000000024, "observed": true, "imputed": false}
36
+ {"eval_index": 35, "rl_env_steps": 701757, "target_step": 700000, "sr": 0.54, "round": 88, "update_index": 264000, "policy_version": 88, "chunk_transitions": 88000, "gradient_updates": 264000, "n_success": 108, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.538, "gain": 0.013500000000000068, "observed": true, "imputed": false}
37
+ {"eval_index": 36, "rl_env_steps": 725695, "target_step": 720000, "sr": 0.53, "round": 91, "update_index": 273000, "policy_version": 91, "chunk_transitions": 91000, "gradient_updates": 273000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5345000000000001, "gain": -0.0004999999999998339, "observed": true, "imputed": false}
38
+ {"eval_index": 37, "rl_env_steps": 741618, "target_step": 740000, "sr": 0.475, "round": 93, "update_index": 279000, "policy_version": 93, "chunk_transitions": 93000, "gradient_updates": 279000, "n_success": 95, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.527, "gain": -0.010000000000000009, "observed": true, "imputed": false}
39
+ {"eval_index": 38, "rl_env_steps": 765539, "target_step": 760000, "sr": 0.475, "round": 96, "update_index": 288000, "policy_version": 96, "chunk_transitions": 96000, "gradient_updates": 288000, "n_success": 95, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5169999999999999, "gain": -0.02300000000000013, "observed": true, "imputed": false}
40
+ {"eval_index": 39, "rl_env_steps": 781473, "target_step": 780000, "sr": 0.56, "round": 98, "update_index": 294000, "policy_version": 98, "chunk_transitions": 98000, "gradient_updates": 294000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5199999999999999, "gain": -0.019500000000000184, "observed": true, "imputed": false}
41
+ {"eval_index": 40, "rl_env_steps": 805392, "target_step": 800000, "sr": 0.58, "round": 101, "update_index": 303000, "policy_version": 101, "chunk_transitions": 101000, "gradient_updates": 303000, "n_success": 116, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5285, "gain": -0.0015000000000000568, "observed": true, "imputed": false}
42
+ {"eval_index": 41, "rl_env_steps": 821308, "target_step": 820000, "sr": 0.625, "round": 103, "update_index": 309000, "policy_version": 103, "chunk_transitions": 103000, "gradient_updates": 309000, "n_success": 125, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.537, "gain": 0.0010000000000000009, "observed": true, "imputed": false}
43
+ {"eval_index": 42, "rl_env_steps": 845252, "target_step": 840000, "sr": 0.48, "round": 106, "update_index": 318000, "policy_version": 106, "chunk_transitions": 106000, "gradient_updates": 318000, "n_success": 96, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5359999999999999, "gain": -0.00550000000000006, "observed": true, "imputed": false}
44
+ {"eval_index": 43, "rl_env_steps": 861186, "target_step": 860000, "sr": 0.555, "round": 108, "update_index": 324000, "policy_version": 108, "chunk_transitions": 108000, "gradient_updates": 324000, "n_success": 111, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.538, "gain": -0.0040000000000000036, "observed": true, "imputed": false}
45
+ {"eval_index": 44, "rl_env_steps": 885110, "target_step": 880000, "sr": 0.575, "round": 111, "update_index": 333000, "policy_version": 111, "chunk_transitions": 111000, "gradient_updates": 333000, "n_success": 115, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5395000000000001, "gain": -0.0030000000000000027, "observed": true, "imputed": false}
46
+ {"eval_index": 45, "rl_env_steps": 901032, "target_step": 900000, "sr": 0.51, "round": 113, "update_index": 339000, "policy_version": 113, "chunk_transitions": 113000, "gradient_updates": 339000, "n_success": 102, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5365, "gain": -0.0015000000000000568, "observed": true, "imputed": false}
47
+ {"eval_index": 46, "rl_env_steps": 924952, "target_step": 920000, "sr": 0.55, "round": 116, "update_index": 348000, "policy_version": 116, "chunk_transitions": 116000, "gradient_updates": 348000, "n_success": 110, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5384999999999999, "gain": 0.0039999999999997815, "observed": true, "imputed": false}
48
+ {"eval_index": 47, "rl_env_steps": 940913, "target_step": 940000, "sr": 0.56, "round": 118, "update_index": 354000, "policy_version": 118, "chunk_transitions": 118000, "gradient_updates": 354000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.547, "gain": 0.020000000000000018, "observed": true, "imputed": false}
49
+ {"eval_index": 48, "rl_env_steps": 964840, "target_step": 960000, "sr": 0.545, "round": 121, "update_index": 363000, "policy_version": 121, "chunk_transitions": 121000, "gradient_updates": 363000, "n_success": 109, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.554, "gain": 0.037000000000000144, "observed": true, "imputed": false}
50
+ {"eval_index": 49, "rl_env_steps": 980787, "target_step": 980000, "sr": 0.475, "round": 123, "update_index": 369000, "policy_version": 123, "chunk_transitions": 123000, "gradient_updates": 369000, "n_success": 95, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5455, "gain": 0.025500000000000078, "observed": true, "imputed": false}
51
+ {"eval_index": 50, "rl_env_steps": 1004729, "target_step": 1000000, "sr": 0.53, "round": 126, "update_index": 378000, "policy_version": 126, "chunk_transitions": 126000, "gradient_updates": 378000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5405, "gain": 0.01200000000000001, "observed": true, "imputed": false}
52
+ {"eval_index": 51, "rl_env_steps": 1020661, "target_step": 1020000, "sr": 0.54, "round": 128, "update_index": 384000, "policy_version": 128, "chunk_transitions": 128000, "gradient_updates": 384000, "n_success": 108, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.532, "gain": -0.0050000000000000044, "observed": true, "imputed": false}
53
+ {"eval_index": 52, "rl_env_steps": 1044581, "target_step": 1040000, "sr": 0.525, "round": 131, "update_index": 393000, "policy_version": 131, "chunk_transitions": 131000, "gradient_updates": 393000, "n_success": 105, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5365, "gain": 0.000500000000000056, "observed": true, "imputed": false}
54
+ {"eval_index": 53, "rl_env_steps": 1060525, "target_step": 1060000, "sr": 0.475, "round": 133, "update_index": 399000, "policy_version": 133, "chunk_transitions": 133000, "gradient_updates": 399000, "n_success": 95, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5285, "gain": -0.009500000000000064, "observed": true, "imputed": false}
55
+ {"eval_index": 54, "rl_env_steps": 1084427, "target_step": 1080000, "sr": 0.535, "round": 136, "update_index": 408000, "policy_version": 136, "chunk_transitions": 136000, "gradient_updates": 408000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5245, "gain": -0.015000000000000124, "observed": true, "imputed": false}
56
+ {"eval_index": 55, "rl_env_steps": 1100393, "target_step": 1100000, "sr": 0.56, "round": 138, "update_index": 414000, "policy_version": 138, "chunk_transitions": 138000, "gradient_updates": 414000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5295, "gain": -0.007000000000000006, "observed": true, "imputed": false}
57
+ {"eval_index": 56, "rl_env_steps": 1124301, "target_step": 1120000, "sr": 0.615, "round": 141, "update_index": 423000, "policy_version": 141, "chunk_transitions": 141000, "gradient_updates": 423000, "n_success": 123, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5360000000000001, "gain": -0.0024999999999997247, "observed": true, "imputed": false}
58
+ {"eval_index": 57, "rl_env_steps": 1140253, "target_step": 1140000, "sr": 0.545, "round": 143, "update_index": 429000, "policy_version": 143, "chunk_transitions": 143000, "gradient_updates": 429000, "n_success": 109, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5345000000000001, "gain": -0.012499999999999956, "observed": true, "imputed": false}
59
+ {"eval_index": 58, "rl_env_steps": 1164166, "target_step": 1160000, "sr": 0.435, "round": 146, "update_index": 438000, "policy_version": 146, "chunk_transitions": 146000, "gradient_updates": 438000, "n_success": 87, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5235, "gain": -0.030500000000000083, "observed": true, "imputed": false}
60
+ {"eval_index": 59, "rl_env_steps": 1180098, "target_step": 1180000, "sr": 0.555, "round": 148, "update_index": 444000, "policy_version": 148, "chunk_transitions": 148000, "gradient_updates": 444000, "n_success": 111, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5315000000000001, "gain": -0.013999999999999901, "observed": true, "imputed": false}
61
+ {"eval_index": 60, "rl_env_steps": 1203983, "target_step": 1200000, "sr": 0.52, "round": 151, "update_index": 453000, "policy_version": 151, "chunk_transitions": 151000, "gradient_updates": 453000, "n_success": 104, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5305, "gain": -0.010000000000000009, "observed": true, "imputed": false}
62
+ {"eval_index": 61, "rl_env_steps": 1227895, "target_step": 1220000, "sr": 0.59, "round": 154, "update_index": 462000, "policy_version": 154, "chunk_transitions": 154000, "gradient_updates": 462000, "n_success": 118, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5355000000000001, "gain": 0.0035000000000000586, "observed": true, "imputed": false}
63
+ {"eval_index": 62, "rl_env_steps": 1243838, "target_step": 1240000, "sr": 0.465, "round": 156, "update_index": 468000, "policy_version": 156, "chunk_transitions": 156000, "gradient_updates": 468000, "n_success": 93, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5295, "gain": -0.007000000000000006, "observed": true, "imputed": false}
64
+ {"eval_index": 63, "rl_env_steps": 1267755, "target_step": 1260000, "sr": 0.54, "round": 159, "update_index": 477000, "policy_version": 159, "chunk_transitions": 159000, "gradient_updates": 477000, "n_success": 108, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.536, "gain": 0.007500000000000062, "observed": true, "imputed": false}
65
+ {"eval_index": 64, "rl_env_steps": 1283705, "target_step": 1280000, "sr": 0.495, "round": 161, "update_index": 483000, "policy_version": 161, "chunk_transitions": 161000, "gradient_updates": 483000, "n_success": 99, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.532, "gain": 0.007500000000000062, "observed": true, "imputed": false}
66
+ {"eval_index": 65, "rl_env_steps": 1307623, "target_step": 1300000, "sr": 0.56, "round": 164, "update_index": 492000, "policy_version": 164, "chunk_transitions": 164000, "gradient_updates": 492000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.532, "gain": 0.0025000000000000577, "observed": true, "imputed": false}
67
+ {"eval_index": 66, "rl_env_steps": 1323547, "target_step": 1320000, "sr": 0.52, "round": 166, "update_index": 498000, "policy_version": 166, "chunk_transitions": 166000, "gradient_updates": 498000, "n_success": 104, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5225, "gain": -0.013500000000000179, "observed": true, "imputed": false}
68
+ {"eval_index": 67, "rl_env_steps": 1347470, "target_step": 1340000, "sr": 0.58, "round": 169, "update_index": 507000, "policy_version": 169, "chunk_transitions": 169000, "gradient_updates": 507000, "n_success": 116, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.526, "gain": -0.008500000000000063, "observed": true, "imputed": false}
69
+ {"eval_index": 68, "rl_env_steps": 1363417, "target_step": 1360000, "sr": 0.535, "round": 171, "update_index": 513000, "policy_version": 171, "chunk_transitions": 171000, "gradient_updates": 513000, "n_success": 107, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.536, "gain": 0.012500000000000067, "observed": true, "imputed": false}
70
+ {"eval_index": 69, "rl_env_steps": 1387336, "target_step": 1380000, "sr": 0.61, "round": 174, "update_index": 522000, "policy_version": 174, "chunk_transitions": 174000, "gradient_updates": 522000, "n_success": 122, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5415000000000001, "gain": 0.010000000000000009, "observed": true, "imputed": false}
71
+ {"eval_index": 70, "rl_env_steps": 1403261, "target_step": 1400000, "sr": 0.53, "round": 176, "update_index": 528000, "policy_version": 176, "chunk_transitions": 176000, "gradient_updates": 528000, "n_success": 106, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5425000000000001, "gain": 0.012000000000000122, "observed": true, "imputed": false}
72
+ {"eval_index": 71, "rl_env_steps": 1427155, "target_step": 1420000, "sr": 0.6, "round": 179, "update_index": 537000, "policy_version": 179, "chunk_transitions": 179000, "gradient_updates": 537000, "n_success": 120, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5435000000000001, "gain": 0.008000000000000007, "observed": true, "imputed": false}
73
+ {"eval_index": 72, "rl_env_steps": 1443113, "target_step": 1440000, "sr": 0.555, "round": 181, "update_index": 543000, "policy_version": 181, "chunk_transitions": 181000, "gradient_updates": 543000, "n_success": 111, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5525, "gain": 0.02300000000000002, "observed": true, "imputed": false}
74
+ {"eval_index": 73, "rl_env_steps": 1467047, "target_step": 1460000, "sr": 0.55, "round": 184, "update_index": 552000, "policy_version": 184, "chunk_transitions": 184000, "gradient_updates": 552000, "n_success": 110, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5534999999999999, "gain": 0.01749999999999985, "observed": true, "imputed": false}
75
+ {"eval_index": 74, "rl_env_steps": 1482993, "target_step": 1480000, "sr": 0.565, "round": 186, "update_index": 558000, "policy_version": 186, "chunk_transitions": 186000, "gradient_updates": 558000, "n_success": 113, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5605, "gain": 0.02849999999999997, "observed": true, "imputed": false}
76
+ {"eval_index": 75, "rl_env_steps": 1506902, "target_step": 1500000, "sr": 0.55, "round": 189, "update_index": 567000, "policy_version": 189, "chunk_transitions": 189000, "gradient_updates": 567000, "n_success": 110, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5595, "gain": 0.02749999999999997, "observed": true, "imputed": false}
77
+ {"eval_index": 76, "rl_env_steps": 1522866, "target_step": 1520000, "sr": 0.51, "round": 191, "update_index": 573000, "policy_version": 191, "chunk_transitions": 191000, "gradient_updates": 573000, "n_success": 102, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5585, "gain": 0.03600000000000003, "observed": true, "imputed": false}
78
+ {"eval_index": 77, "rl_env_steps": 1546745, "target_step": 1540000, "sr": 0.56, "round": 194, "update_index": 582000, "policy_version": 194, "chunk_transitions": 194000, "gradient_updates": 582000, "n_success": 112, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5565, "gain": 0.03049999999999997, "observed": true, "imputed": false}
79
+ {"eval_index": 78, "rl_env_steps": 1562702, "target_step": 1560000, "sr": 0.55, "round": 196, "update_index": 588000, "policy_version": 196, "chunk_transitions": 196000, "gradient_updates": 588000, "n_success": 110, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5579999999999999, "gain": 0.02199999999999991, "observed": true, "imputed": false}
80
+ {"eval_index": 79, "rl_env_steps": 1586613, "target_step": 1580000, "sr": 0.5, "round": 199, "update_index": 597000, "policy_version": 199, "chunk_transitions": 199000, "gradient_updates": 597000, "n_success": 100, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5469999999999999, "gain": 0.005499999999999838, "observed": true, "imputed": false}
81
+ {"eval_index": 80, "rl_env_steps": 1602575, "target_step": 1600000, "sr": 0.54, "round": 201, "update_index": 603000, "policy_version": 201, "chunk_transitions": 201000, "gradient_updates": 603000, "n_success": 108, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.5479999999999999, "gain": 0.005499999999999838, "observed": true, "imputed": false}
RL/Transport50/QC/diverse50/seed_2/complete/README.md ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ Complete run archive
2
+
3
+ See publication.json for identity and terminal state. Download all files in this complete/ directory, then run `python restore.py --archive-dir . --output /path/to/restored`. It checks part hashes and every restored file against manifest.json. zstd command is required for .zst archives.
4
+
5
+ The archive contains run/ (all retained checkpoints, replay, offline buffer, metrics, evaluations, diagnostics and W&B records) and dependencies/ (base BC, manifests, exact observed source and runtime inventory). Original absolute paths remain in historical metadata. Use original directory locations or an explicit recorded path migration before training. Raw demo/image datasets are inputs referenced by their recorded hashes; they are not duplicated in this run-artifact archive. Broken external W&B service-log links are enumerated, never represented as archived bytes.
RL/Transport50/QC/diverse50/seed_2/complete/SHA256SUMS ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ e2d27cdeb38ac6d79f3d06d58617d1c07f996dcf5f27f1a165223ab9e9ee06a5 best/source_metadata.json
2
+ dcfad10086f13bf22dccfb4b3ceccb8f4a572f585a603e835f3cac8ce9d9a98d best/source_resolved_config.json
3
+ c90217e70689435f766bfc69fa734ffa61175b8cc6c5ee2559faf940490cce49 best/state.pkl
4
+ 03ac00068b2d17157cf81521c289c5fb5617dcbb19ca9035e66260783b7fb031 comparison_evaluations.jsonl
5
+ 39c57050e748e4d3f5fd0314a47c05e329d07d4468132cce324735f7fef59b56 complete/README.md
6
+ ac4ede7c330aac53c999e095d6411f7732686732a907b92bf06ab85c4e709b08 complete/archive.json
7
+ 01f16336f2d21bcb9ed6ebb8f8871ed51a0f8fc63e8ec602df3d0c3bd013888a complete/full_run.tar.zst.part000
8
+ 11d6d63d7f20ef02748f1c98e7f88580f8de86fe72d00d57177c87684b04324f complete/full_run.tar.zst.part001
9
+ 18193257d9f772b4d57713bc7d4cecb13417c73f52614a4edd6d97d78f116369 complete/full_run.tar.zst.part002
10
+ 93a7288a6fe8a275a2f504bb96b7aa7a1d969e73114280b5829d707a8038933b complete/full_run.tar.zst.part003
11
+ 6c23b66123abaf97eb02943ba88cd70e08a7a3854a3584eede4da181d3bc0ec1 complete/full_run.tar.zst.part004
12
+ 4910609c537aeb7f04adf7a6b5adbba21743d1e15f779fc61e63b86741c66616 complete/local_archive_verification.json
13
+ 3912317b2dc47846bea4c7a24296a1e20bee42ffc9041fedca0547b3d077d6c5 complete/manifest.json
14
+ 933f3e2a343496808f7409a9a4988ac9ac13efec9b24dcc638f4c00a1c3c09d8 complete/publication.json
15
+ 149a44cd0b002e83f507d1e46d813d5e2b8cf6f9e1bb51f14708343f48788ac2 complete/restore.py
16
+ 7a3364e9d3b59abd99ffa952a38b56db495a5f4c4b5df9bc2000cba93706e46a convergence.json
17
+ a206a72c97a1f41560eb77cbab7611c887893a00c1a4adaeda070626ae26f689 convergence.jsonl
18
+ 1b25379844a840f7d57b232050ca341ae3c5b8c81c66394f04dc4e37f5ec9b6e eval/final.parquet
19
+ 9b60ce05b077ddaff3a769b328b1a281d4556bc88906d756b8973fe316a5eba0 final/source_metadata.json
20
+ dcfad10086f13bf22dccfb4b3ceccb8f4a572f585a603e835f3cac8ce9d9a98d final/source_resolved_config.json
21
+ e136b372e5d08e8a0dc41d409398c01a57982fde2ece53aa12553e59923cc764 final/state.pkl
22
+ 1cc61d3c136149ec10a2aaf7cdd4546df842bdf04b77a103e603129e57b816c1 frozen_base.json
23
+ c656fa57ecd8081742849290d5e80dc8f94f8c331442fc5e1d6e5636b590ff97 offline_buffer.json
24
+ dcfad10086f13bf22dccfb4b3ceccb8f4a572f585a603e835f3cac8ce9d9a98d resolved_config.json
25
+ 2d9edea94d4387b23cbc8fe556d9b2676bce08d82992f2b2295765f00dcb527e start_protocol.json
26
+ eccb70900f4efb70ffae4ba2a8320bc3647c89504894b072f8b895c054fe8dee status.json
27
+ c5f2157a1ed468d784ab9eb326bc075b1bf7bea8b46981a1fd7d26aa9010f112 training_accounting.json
RL/Transport50/QC/diverse50/seed_2/complete/archive.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "format": "tar.zst",
3
+ "parts": [
4
+ {
5
+ "path": "full_run.tar.zst.part000",
6
+ "size": 1073741824,
7
+ "sha256": "01f16336f2d21bcb9ed6ebb8f8871ed51a0f8fc63e8ec602df3d0c3bd013888a"
8
+ },
9
+ {
10
+ "path": "full_run.tar.zst.part001",
11
+ "size": 1073741824,
12
+ "sha256": "11d6d63d7f20ef02748f1c98e7f88580f8de86fe72d00d57177c87684b04324f"
13
+ },
14
+ {
15
+ "path": "full_run.tar.zst.part002",
16
+ "size": 1073741824,
17
+ "sha256": "18193257d9f772b4d57713bc7d4cecb13417c73f52614a4edd6d97d78f116369"
18
+ },
19
+ {
20
+ "path": "full_run.tar.zst.part003",
21
+ "size": 1073741824,
22
+ "sha256": "93a7288a6fe8a275a2f504bb96b7aa7a1d969e73114280b5829d707a8038933b"
23
+ },
24
+ {
25
+ "path": "full_run.tar.zst.part004",
26
+ "size": 874241244,
27
+ "sha256": "6c23b66123abaf97eb02943ba88cd70e08a7a3854a3584eede4da181d3bc0ec1"
28
+ }
29
+ ],
30
+ "manifest_sha256": "3912317b2dc47846bea4c7a24296a1e20bee42ffc9041fedca0547b3d077d6c5",
31
+ "hardlink_content_dedup": true,
32
+ "member_count": 6439
33
+ }
RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part000 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01f16336f2d21bcb9ed6ebb8f8871ed51a0f8fc63e8ec602df3d0c3bd013888a
3
+ size 1073741824
RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part001 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11d6d63d7f20ef02748f1c98e7f88580f8de86fe72d00d57177c87684b04324f
3
+ size 1073741824
RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part002 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:18193257d9f772b4d57713bc7d4cecb13417c73f52614a4edd6d97d78f116369
3
+ size 1073741824
RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part003 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93a7288a6fe8a275a2f504bb96b7aa7a1d969e73114280b5829d707a8038933b
3
+ size 1073741824
RL/Transport50/QC/diverse50/seed_2/complete/full_run.tar.zst.part004 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c23b66123abaf97eb02943ba88cd70e08a7a3854a3584eede4da181d3bc0ec1
3
+ size 874241244
RL/Transport50/QC/diverse50/seed_2/complete/local_archive_verification.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "passed": true,
3
+ "verified_members": 6438,
4
+ "verified_archive_parts": 5,
5
+ "time": 1793313521.8101406
6
+ }
RL/Transport50/QC/diverse50/seed_2/complete/manifest.json ADDED
The diff for this file is too large to render. See raw diff
 
RL/Transport50/QC/diverse50/seed_2/complete/publication.json ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 2,
3
+ "run_id": "15e00de4",
4
+ "task": "Transport50",
5
+ "algorithm": "QC",
6
+ "variant": "qc",
7
+ "cohort": "diverse50",
8
+ "seed": 2,
9
+ "protocol": "study2_protocol_v10",
10
+ "status": "budget_complete",
11
+ "path": "RL/Transport50/QC/diverse50/seed_2",
12
+ "checkpoints": {
13
+ "best": {
14
+ "path": "RL/Transport50/QC/diverse50/seed_2/best/state.pkl",
15
+ "sha256": "c90217e70689435f766bfc69fa734ffa61175b8cc6c5ee2559faf940490cce49",
16
+ "bytes": 214992763,
17
+ "source_path": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261028_transport_diverse50_qc_v10_seed2/checkpoints/best/eval_00041_step_821308_session_1793131010203021286/checkpoint/state.pkl",
18
+ "evaluation": {
19
+ "chunk_transitions": 103000,
20
+ "episodes": 200,
21
+ "eval_index": 41,
22
+ "gain": 0.0010000000000000009,
23
+ "gradient_updates": 309000,
24
+ "ma": 0.537,
25
+ "n_success": 125,
26
+ "policy_version": 103,
27
+ "rl_env_steps": 821308,
28
+ "round": 103,
29
+ "seed_start": 960000,
30
+ "sr": 0.625,
31
+ "success_requested": 0,
32
+ "success_requested_imitation": 0,
33
+ "success_rounds_with_skips": 0,
34
+ "success_skipped": 0,
35
+ "success_skipped_imitation": 0,
36
+ "success_skipped_td": 0,
37
+ "target_step": 820000,
38
+ "update_index": 309000
39
+ }
40
+ },
41
+ "final": {
42
+ "path": "RL/Transport50/QC/diverse50/seed_2/final/state.pkl",
43
+ "sha256": "e136b372e5d08e8a0dc41d409398c01a57982fde2ece53aa12553e59923cc764",
44
+ "bytes": 215002264,
45
+ "source_path": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261028_transport_diverse50_qc_v10_seed2/checkpoints/final/session_1793131010203021286/checkpoint/state.pkl",
46
+ "evaluation": {
47
+ "episodes": 300,
48
+ "imputed": false,
49
+ "n_success": 156,
50
+ "observed": true,
51
+ "seed_start": 800000,
52
+ "sr": 0.52
53
+ }
54
+ }
55
+ },
56
+ "complete_artifacts": "RL/Transport50/QC/diverse50/seed_2/complete",
57
+ "source_host": "Paratera",
58
+ "source_run": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261028_transport_diverse50_qc_v10_seed2",
59
+ "artifact_files": 6438,
60
+ "archive_bytes": 5169208540,
61
+ "omitted_broken_log_links": 0
62
+ }
RL/Transport50/QC/diverse50/seed_2/complete/restore.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Restore a full RL run and verify every archived member without executing its contents."""
2
+ import argparse,hashlib,json,os,subprocess,tarfile,threading,shutil
3
+ from pathlib import Path
4
+
5
+ def digest(p):
6
+ h=hashlib.sha256()
7
+ with p.open('rb') as f:
8
+ for b in iter(lambda:f.read(8*1024**2),b''):h.update(b)
9
+ return h.hexdigest()
10
+ def main():
11
+ ap=argparse.ArgumentParser();ap.add_argument('--archive-dir',type=Path,default=Path('.'));ap.add_argument('--output',type=Path,required=True);args=ap.parse_args();base=args.archive_dir;out=args.output.resolve();out.mkdir(parents=True,exist_ok=True);assert not any(out.iterdir()),'Use an empty output directory'
12
+ arc=json.loads((base/'archive.json').read_text());assert digest(base/'manifest.json')==arc['manifest_sha256'];manifest=json.loads((base/'manifest.json').read_text());expected={e['path']:e for e in manifest['entries']}
13
+ for part in arc['parts']:
14
+ p=base/part['path'];assert p.stat().st_size==part['size'] and digest(p)==part['sha256'],p
15
+ cmd=['zstd','-q','-d','-c'] if arc['format'].endswith('zst') else ['gzip','-d','-c'];proc=subprocess.Popen(cmd,stdin=subprocess.PIPE,stdout=subprocess.PIPE);errors=[]
16
+ def feed():
17
+ try:
18
+ for part in arc['parts']:
19
+ with (base/part['path']).open('rb') as f:shutil.copyfileobj(f,proc.stdin,1024**2)
20
+ except BaseException as e:errors.append(e)
21
+ finally:proc.stdin.close()
22
+ t=threading.Thread(target=feed);t.start();seen={}
23
+ with tarfile.open(fileobj=proc.stdout,mode='r|') as tar:
24
+ for m in tar:
25
+ if m.name=='MANIFEST.json':assert hashlib.sha256(tar.extractfile(m).read()).hexdigest()==arc['manifest_sha256'];continue
26
+ e=expected[m.name];dest=out/m.name;assert dest.resolve().is_relative_to(out);dest.parent.mkdir(parents=True,exist_ok=True)
27
+ if m.issym():
28
+ assert e['type']=='symlink' and m.linkname==e['target'] and (dest.parent/m.linkname).resolve().is_relative_to(out);dest.symlink_to(m.linkname);seen[m.name]=None
29
+ elif m.islnk():
30
+ assert seen[m.linkname]==e['sha256'];src=out/m.linkname;assert src.resolve().is_relative_to(out);os.link(src,dest);seen[m.name]=e['sha256']
31
+ else:
32
+ assert m.isfile() and m.size==e['size'];h=hashlib.sha256()
33
+ with dest.open('xb') as f:
34
+ inp=tar.extractfile(m)
35
+ for b in iter(lambda:inp.read(8*1024**2),b''):f.write(b);h.update(b)
36
+ assert h.hexdigest()==e['sha256'],m.name;dest.chmod(e.get('mode',0o644));seen[m.name]=h.hexdigest()
37
+ t.join();assert proc.wait()==0 and not errors and len(seen)==len(expected);shutil.copy2(base/'manifest.json',out/'MANIFEST.json');(out/'RESTORE_VERIFIED.json').write_text(json.dumps({'passed':True,'files':len(seen),'run_id':manifest['run']['run_id']},indent=2));print('RESTORE VERIFIED',manifest['run']['run_id'],len(seen),'members')
38
+ if __name__=='__main__':main()
RL/Transport50/QC/diverse50/seed_2/convergence.json ADDED
@@ -0,0 +1,367 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "evals": 81,
3
+ "final_eval": {
4
+ "episodes": 300,
5
+ "imputed": false,
6
+ "n_success": 156,
7
+ "observed": true,
8
+ "seed_start": 800000,
9
+ "sr": 0.52,
10
+ "successes": [
11
+ 1,
12
+ 1,
13
+ 0,
14
+ 0,
15
+ 1,
16
+ 0,
17
+ 1,
18
+ 0,
19
+ 0,
20
+ 1,
21
+ 1,
22
+ 0,
23
+ 1,
24
+ 0,
25
+ 1,
26
+ 1,
27
+ 0,
28
+ 0,
29
+ 0,
30
+ 1,
31
+ 1,
32
+ 0,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 1,
44
+ 0,
45
+ 1,
46
+ 1,
47
+ 0,
48
+ 0,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 0,
54
+ 1,
55
+ 0,
56
+ 1,
57
+ 1,
58
+ 0,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 0,
63
+ 1,
64
+ 0,
65
+ 1,
66
+ 1,
67
+ 0,
68
+ 1,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 1,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 1,
78
+ 0,
79
+ 1,
80
+ 0,
81
+ 1,
82
+ 0,
83
+ 0,
84
+ 0,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 0,
90
+ 1,
91
+ 0,
92
+ 1,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 1,
98
+ 0,
99
+ 0,
100
+ 0,
101
+ 1,
102
+ 0,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 0,
107
+ 0,
108
+ 0,
109
+ 1,
110
+ 1,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 0,
118
+ 1,
119
+ 0,
120
+ 0,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 0,
127
+ 0,
128
+ 1,
129
+ 0,
130
+ 1,
131
+ 1,
132
+ 0,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 0,
146
+ 0,
147
+ 1,
148
+ 1,
149
+ 0,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 0,
156
+ 0,
157
+ 1,
158
+ 1,
159
+ 0,
160
+ 1,
161
+ 0,
162
+ 1,
163
+ 0,
164
+ 0,
165
+ 0,
166
+ 0,
167
+ 0,
168
+ 0,
169
+ 1,
170
+ 1,
171
+ 0,
172
+ 1,
173
+ 1,
174
+ 1,
175
+ 0,
176
+ 1,
177
+ 0,
178
+ 1,
179
+ 0,
180
+ 1,
181
+ 1,
182
+ 0,
183
+ 1,
184
+ 0,
185
+ 0,
186
+ 0,
187
+ 1,
188
+ 1,
189
+ 1,
190
+ 0,
191
+ 1,
192
+ 1,
193
+ 0,
194
+ 1,
195
+ 0,
196
+ 1,
197
+ 1,
198
+ 1,
199
+ 0,
200
+ 1,
201
+ 1,
202
+ 1,
203
+ 0,
204
+ 1,
205
+ 1,
206
+ 0,
207
+ 0,
208
+ 0,
209
+ 1,
210
+ 0,
211
+ 0,
212
+ 0,
213
+ 0,
214
+ 0,
215
+ 0,
216
+ 0,
217
+ 0,
218
+ 0,
219
+ 1,
220
+ 0,
221
+ 1,
222
+ 1,
223
+ 0,
224
+ 1,
225
+ 1,
226
+ 1,
227
+ 1,
228
+ 1,
229
+ 0,
230
+ 1,
231
+ 1,
232
+ 0,
233
+ 1,
234
+ 0,
235
+ 0,
236
+ 1,
237
+ 0,
238
+ 1,
239
+ 0,
240
+ 1,
241
+ 0,
242
+ 0,
243
+ 0,
244
+ 1,
245
+ 1,
246
+ 0,
247
+ 0,
248
+ 0,
249
+ 0,
250
+ 0,
251
+ 1,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 1,
256
+ 0,
257
+ 0,
258
+ 1,
259
+ 1,
260
+ 0,
261
+ 1,
262
+ 1,
263
+ 0,
264
+ 1,
265
+ 0,
266
+ 1,
267
+ 1,
268
+ 1,
269
+ 0,
270
+ 1,
271
+ 1,
272
+ 0,
273
+ 0,
274
+ 1,
275
+ 0,
276
+ 1,
277
+ 0,
278
+ 0,
279
+ 0,
280
+ 1,
281
+ 0,
282
+ 1,
283
+ 1,
284
+ 1,
285
+ 0,
286
+ 0,
287
+ 1,
288
+ 1,
289
+ 1,
290
+ 1,
291
+ 1,
292
+ 1,
293
+ 0,
294
+ 0,
295
+ 0,
296
+ 0,
297
+ 0,
298
+ 0,
299
+ 0,
300
+ 0,
301
+ 1,
302
+ 1,
303
+ 0,
304
+ 1,
305
+ 1,
306
+ 0,
307
+ 0,
308
+ 1,
309
+ 1,
310
+ 1
311
+ ]
312
+ },
313
+ "last": {
314
+ "chunk_transitions": 201000,
315
+ "episodes": 200,
316
+ "eval_index": 80,
317
+ "gain": 0.005499999999999838,
318
+ "gradient_updates": 603000,
319
+ "ma": 0.5479999999999999,
320
+ "n_success": 108,
321
+ "policy_version": 201,
322
+ "rl_env_steps": 1602575,
323
+ "round": 201,
324
+ "seed_start": 960000,
325
+ "sr": 0.54,
326
+ "success_requested": 0,
327
+ "success_requested_imitation": 0,
328
+ "success_rounds_with_skips": 0,
329
+ "success_skipped": 0,
330
+ "success_skipped_imitation": 0,
331
+ "success_skipped_td": 0,
332
+ "target_step": 1600000,
333
+ "update_index": 603000
334
+ },
335
+ "rule": {
336
+ "early_stop": false,
337
+ "every_env_steps": 20000,
338
+ "gain_lag": 10,
339
+ "high_sr": 0.95,
340
+ "high_sr_evals": 5,
341
+ "include_step0": true,
342
+ "ma_window": 10,
343
+ "max_env_steps": 1606196,
344
+ "min_env_steps": 200000,
345
+ "min_gain": 0.02,
346
+ "zero_success_evals": 8
347
+ },
348
+ "status": "budget_complete",
349
+ "stop": {
350
+ "eval_index": 80,
351
+ "gain": 0.005499999999999838,
352
+ "ma": 0.5479999999999999,
353
+ "rl_env_steps": 1610541,
354
+ "sr": 0.54,
355
+ "status": "budget_complete",
356
+ "trend": "flat"
357
+ },
358
+ "would_stop": {
359
+ "eval_index": 19,
360
+ "gain": -0.004500000000000004,
361
+ "ma": 0.49800000000000005,
362
+ "rl_env_steps": 382870,
363
+ "sr": 0.535,
364
+ "status": "converged_plateau",
365
+ "trend": "flat"
366
+ }
367
+ }
RL/Transport50/QC/diverse50/seed_2/convergence.jsonl ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"chunk_transitions": 0, "episodes": 200, "eval_index": 0, "gain": null, "gradient_updates": 0, "ma": null, "n_success": 101, "policy_version": 0, "rl_env_steps": 0, "round": 0, "seed_start": 960000, "sr": 0.505, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 0, "update_index": 0}
2
+ {"chunk_transitions": 3000, "episodes": 200, "eval_index": 1, "gain": null, "gradient_updates": 9000, "ma": null, "n_success": 92, "policy_version": 3, "rl_env_steps": 23934, "round": 3, "seed_start": 960000, "sr": 0.46, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 20000, "update_index": 9000}
3
+ {"chunk_transitions": 6000, "episodes": 200, "eval_index": 2, "gain": null, "gradient_updates": 18000, "ma": null, "n_success": 98, "policy_version": 6, "rl_env_steps": 47852, "round": 6, "seed_start": 960000, "sr": 0.49, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 40000, "update_index": 18000}
4
+ {"chunk_transitions": 8000, "episodes": 200, "eval_index": 3, "gain": null, "gradient_updates": 24000, "ma": null, "n_success": 82, "policy_version": 8, "rl_env_steps": 63816, "round": 8, "seed_start": 960000, "sr": 0.41, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 60000, "update_index": 24000}
5
+ {"chunk_transitions": 11000, "episodes": 200, "eval_index": 4, "gain": null, "gradient_updates": 33000, "ma": null, "n_success": 107, "policy_version": 11, "rl_env_steps": 87736, "round": 11, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 80000, "update_index": 33000}
6
+ {"chunk_transitions": 13000, "episodes": 200, "eval_index": 5, "gain": null, "gradient_updates": 39000, "ma": null, "n_success": 99, "policy_version": 13, "rl_env_steps": 103698, "round": 13, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 100000, "update_index": 39000}
7
+ {"chunk_transitions": 16000, "episodes": 200, "eval_index": 6, "gain": null, "gradient_updates": 48000, "ma": null, "n_success": 107, "policy_version": 16, "rl_env_steps": 127620, "round": 16, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 120000, "update_index": 48000}
8
+ {"chunk_transitions": 18000, "episodes": 200, "eval_index": 7, "gain": null, "gradient_updates": 54000, "ma": null, "n_success": 104, "policy_version": 18, "rl_env_steps": 143590, "round": 18, "seed_start": 960000, "sr": 0.52, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 140000, "update_index": 54000}
9
+ {"chunk_transitions": 21000, "episodes": 200, "eval_index": 8, "gain": null, "gradient_updates": 63000, "ma": null, "n_success": 102, "policy_version": 21, "rl_env_steps": 167527, "round": 21, "seed_start": 960000, "sr": 0.51, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 160000, "update_index": 63000}
10
+ {"chunk_transitions": 23000, "episodes": 200, "eval_index": 9, "gain": null, "gradient_updates": 69000, "ma": 0.5025000000000001, "n_success": 113, "policy_version": 23, "rl_env_steps": 183481, "round": 23, "seed_start": 960000, "sr": 0.565, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 180000, "update_index": 69000}
11
+ {"chunk_transitions": 26000, "episodes": 200, "eval_index": 10, "gain": null, "gradient_updates": 78000, "ma": 0.5015, "n_success": 99, "policy_version": 26, "rl_env_steps": 207405, "round": 26, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 200000, "update_index": 78000}
12
+ {"chunk_transitions": 28000, "episodes": 200, "eval_index": 11, "gain": null, "gradient_updates": 84000, "ma": 0.5065, "n_success": 102, "policy_version": 28, "rl_env_steps": 223339, "round": 28, "seed_start": 960000, "sr": 0.51, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 220000, "update_index": 84000}
13
+ {"chunk_transitions": 31000, "episodes": 200, "eval_index": 12, "gain": null, "gradient_updates": 93000, "ma": 0.5025, "n_success": 90, "policy_version": 31, "rl_env_steps": 247250, "round": 31, "seed_start": 960000, "sr": 0.45, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 240000, "update_index": 93000}
14
+ {"chunk_transitions": 33000, "episodes": 200, "eval_index": 13, "gain": null, "gradient_updates": 99000, "ma": 0.508, "n_success": 93, "policy_version": 33, "rl_env_steps": 263200, "round": 33, "seed_start": 960000, "sr": 0.465, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 260000, "update_index": 99000}
15
+ {"chunk_transitions": 36000, "episodes": 200, "eval_index": 14, "gain": null, "gradient_updates": 108000, "ma": 0.504, "n_success": 99, "policy_version": 36, "rl_env_steps": 287120, "round": 36, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 280000, "update_index": 108000}
16
+ {"chunk_transitions": 38000, "episodes": 200, "eval_index": 15, "gain": null, "gradient_updates": 114000, "ma": 0.504, "n_success": 99, "policy_version": 38, "rl_env_steps": 303073, "round": 38, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 300000, "update_index": 114000}
17
+ {"chunk_transitions": 41000, "episodes": 200, "eval_index": 16, "gain": null, "gradient_updates": 123000, "ma": 0.4965, "n_success": 92, "policy_version": 41, "rl_env_steps": 327018, "round": 41, "seed_start": 960000, "sr": 0.46, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 320000, "update_index": 123000}
18
+ {"chunk_transitions": 43000, "episodes": 200, "eval_index": 17, "gain": null, "gradient_updates": 129000, "ma": 0.49750000000000005, "n_success": 106, "policy_version": 43, "rl_env_steps": 342983, "round": 43, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 340000, "update_index": 129000}
19
+ {"chunk_transitions": 46000, "episodes": 200, "eval_index": 18, "gain": null, "gradient_updates": 138000, "ma": 0.501, "n_success": 109, "policy_version": 46, "rl_env_steps": 366917, "round": 46, "seed_start": 960000, "sr": 0.545, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 360000, "update_index": 138000}
20
+ {"chunk_transitions": 48000, "episodes": 200, "eval_index": 19, "gain": -0.004500000000000004, "gradient_updates": 144000, "ma": 0.49800000000000005, "n_success": 107, "policy_version": 48, "rl_env_steps": 382870, "round": 48, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 380000, "update_index": 144000}
21
+ {"chunk_transitions": 51000, "episodes": 200, "eval_index": 20, "gain": 0.006000000000000116, "gradient_updates": 153000, "ma": 0.5075000000000001, "n_success": 118, "policy_version": 51, "rl_env_steps": 406795, "round": 51, "seed_start": 960000, "sr": 0.59, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 400000, "update_index": 153000}
22
+ {"chunk_transitions": 53000, "episodes": 200, "eval_index": 21, "gain": -0.0020000000000000018, "gradient_updates": 159000, "ma": 0.5045, "n_success": 96, "policy_version": 53, "rl_env_steps": 422759, "round": 53, "seed_start": 960000, "sr": 0.48, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 420000, "update_index": 159000}
23
+ {"chunk_transitions": 56000, "episodes": 200, "eval_index": 22, "gain": 0.000500000000000056, "gradient_updates": 168000, "ma": 0.503, "n_success": 87, "policy_version": 56, "rl_env_steps": 446688, "round": 56, "seed_start": 960000, "sr": 0.435, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 440000, "update_index": 168000}
24
+ {"chunk_transitions": 58000, "episodes": 200, "eval_index": 23, "gain": 0.0014999999999999458, "gradient_updates": 174000, "ma": 0.5095, "n_success": 106, "policy_version": 58, "rl_env_steps": 462657, "round": 58, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 460000, "update_index": 174000}
25
+ {"chunk_transitions": 61000, "episodes": 200, "eval_index": 24, "gain": 0.011500000000000066, "gradient_updates": 183000, "ma": 0.5155000000000001, "n_success": 111, "policy_version": 61, "rl_env_steps": 486562, "round": 61, "seed_start": 960000, "sr": 0.555, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 480000, "update_index": 183000}
26
+ {"chunk_transitions": 63000, "episodes": 200, "eval_index": 25, "gain": 0.020499999999999963, "gradient_updates": 189000, "ma": 0.5245, "n_success": 117, "policy_version": 63, "rl_env_steps": 502524, "round": 63, "seed_start": 960000, "sr": 0.585, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 500000, "update_index": 189000}
27
+ {"chunk_transitions": 66000, "episodes": 200, "eval_index": 26, "gain": 0.03849999999999992, "gradient_updates": 198000, "ma": 0.5349999999999999, "n_success": 113, "policy_version": 66, "rl_env_steps": 526433, "round": 66, "seed_start": 960000, "sr": 0.565, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 520000, "update_index": 198000}
28
+ {"chunk_transitions": 68000, "episodes": 200, "eval_index": 27, "gain": 0.03949999999999998, "gradient_updates": 204000, "ma": 0.537, "n_success": 110, "policy_version": 68, "rl_env_steps": 542357, "round": 68, "seed_start": 960000, "sr": 0.55, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 540000, "update_index": 204000}
29
+ {"chunk_transitions": 71000, "episodes": 200, "eval_index": 28, "gain": 0.039000000000000035, "gradient_updates": 213000, "ma": 0.54, "n_success": 115, "policy_version": 71, "rl_env_steps": 566281, "round": 71, "seed_start": 960000, "sr": 0.575, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 560000, "update_index": 213000}
30
+ {"chunk_transitions": 73000, "episodes": 200, "eval_index": 29, "gain": 0.04150000000000004, "gradient_updates": 219000, "ma": 0.5395000000000001, "n_success": 106, "policy_version": 73, "rl_env_steps": 582252, "round": 73, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 580000, "update_index": 219000}
31
+ {"chunk_transitions": 76000, "episodes": 200, "eval_index": 30, "gain": 0.022499999999999964, "gradient_updates": 228000, "ma": 0.53, "n_success": 99, "policy_version": 76, "rl_env_steps": 606153, "round": 76, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 600000, "update_index": 228000}
32
+ {"chunk_transitions": 78000, "episodes": 200, "eval_index": 31, "gain": 0.031500000000000083, "gradient_updates": 234000, "ma": 0.536, "n_success": 108, "policy_version": 78, "rl_env_steps": 622092, "round": 78, "seed_start": 960000, "sr": 0.54, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 620000, "update_index": 234000}
33
+ {"chunk_transitions": 81000, "episodes": 200, "eval_index": 32, "gain": 0.03849999999999998, "gradient_updates": 243000, "ma": 0.5415, "n_success": 98, "policy_version": 81, "rl_env_steps": 645964, "round": 81, "seed_start": 960000, "sr": 0.49, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 640000, "update_index": 243000}
34
+ {"chunk_transitions": 83000, "episodes": 200, "eval_index": 33, "gain": 0.032500000000000084, "gradient_updates": 249000, "ma": 0.542, "n_success": 107, "policy_version": 83, "rl_env_steps": 661926, "round": 83, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 660000, "update_index": 249000}
35
+ {"chunk_transitions": 86000, "episodes": 200, "eval_index": 34, "gain": 0.027000000000000024, "gradient_updates": 258000, "ma": 0.5425000000000001, "n_success": 112, "policy_version": 86, "rl_env_steps": 685829, "round": 86, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 680000, "update_index": 258000}
36
+ {"chunk_transitions": 88000, "episodes": 200, "eval_index": 35, "gain": 0.013500000000000068, "gradient_updates": 264000, "ma": 0.538, "n_success": 108, "policy_version": 88, "rl_env_steps": 701757, "round": 88, "seed_start": 960000, "sr": 0.54, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 700000, "update_index": 264000}
37
+ {"chunk_transitions": 91000, "episodes": 200, "eval_index": 36, "gain": -0.0004999999999998339, "gradient_updates": 273000, "ma": 0.5345000000000001, "n_success": 106, "policy_version": 91, "rl_env_steps": 725695, "round": 91, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 720000, "update_index": 273000}
38
+ {"chunk_transitions": 93000, "episodes": 200, "eval_index": 37, "gain": -0.010000000000000009, "gradient_updates": 279000, "ma": 0.527, "n_success": 95, "policy_version": 93, "rl_env_steps": 741618, "round": 93, "seed_start": 960000, "sr": 0.475, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 740000, "update_index": 279000}
39
+ {"chunk_transitions": 96000, "episodes": 200, "eval_index": 38, "gain": -0.02300000000000013, "gradient_updates": 288000, "ma": 0.5169999999999999, "n_success": 95, "policy_version": 96, "rl_env_steps": 765539, "round": 96, "seed_start": 960000, "sr": 0.475, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 760000, "update_index": 288000}
40
+ {"chunk_transitions": 98000, "episodes": 200, "eval_index": 39, "gain": -0.019500000000000184, "gradient_updates": 294000, "ma": 0.5199999999999999, "n_success": 112, "policy_version": 98, "rl_env_steps": 781473, "round": 98, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 780000, "update_index": 294000}
41
+ {"chunk_transitions": 101000, "episodes": 200, "eval_index": 40, "gain": -0.0015000000000000568, "gradient_updates": 303000, "ma": 0.5285, "n_success": 116, "policy_version": 101, "rl_env_steps": 805392, "round": 101, "seed_start": 960000, "sr": 0.58, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 800000, "update_index": 303000}
42
+ {"chunk_transitions": 103000, "episodes": 200, "eval_index": 41, "gain": 0.0010000000000000009, "gradient_updates": 309000, "ma": 0.537, "n_success": 125, "policy_version": 103, "rl_env_steps": 821308, "round": 103, "seed_start": 960000, "sr": 0.625, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 820000, "update_index": 309000}
43
+ {"chunk_transitions": 106000, "episodes": 200, "eval_index": 42, "gain": -0.00550000000000006, "gradient_updates": 318000, "ma": 0.5359999999999999, "n_success": 96, "policy_version": 106, "rl_env_steps": 845252, "round": 106, "seed_start": 960000, "sr": 0.48, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 840000, "update_index": 318000}
44
+ {"chunk_transitions": 108000, "episodes": 200, "eval_index": 43, "gain": -0.0040000000000000036, "gradient_updates": 324000, "ma": 0.538, "n_success": 111, "policy_version": 108, "rl_env_steps": 861186, "round": 108, "seed_start": 960000, "sr": 0.555, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 860000, "update_index": 324000}
45
+ {"chunk_transitions": 111000, "episodes": 200, "eval_index": 44, "gain": -0.0030000000000000027, "gradient_updates": 333000, "ma": 0.5395000000000001, "n_success": 115, "policy_version": 111, "rl_env_steps": 885110, "round": 111, "seed_start": 960000, "sr": 0.575, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 880000, "update_index": 333000}
46
+ {"chunk_transitions": 113000, "episodes": 200, "eval_index": 45, "gain": -0.0015000000000000568, "gradient_updates": 339000, "ma": 0.5365, "n_success": 102, "policy_version": 113, "rl_env_steps": 901032, "round": 113, "seed_start": 960000, "sr": 0.51, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 900000, "update_index": 339000}
47
+ {"chunk_transitions": 116000, "episodes": 200, "eval_index": 46, "gain": 0.0039999999999997815, "gradient_updates": 348000, "ma": 0.5384999999999999, "n_success": 110, "policy_version": 116, "rl_env_steps": 924952, "round": 116, "seed_start": 960000, "sr": 0.55, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 920000, "update_index": 348000}
48
+ {"chunk_transitions": 118000, "episodes": 200, "eval_index": 47, "gain": 0.020000000000000018, "gradient_updates": 354000, "ma": 0.547, "n_success": 112, "policy_version": 118, "rl_env_steps": 940913, "round": 118, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 940000, "update_index": 354000}
49
+ {"chunk_transitions": 121000, "episodes": 200, "eval_index": 48, "gain": 0.037000000000000144, "gradient_updates": 363000, "ma": 0.554, "n_success": 109, "policy_version": 121, "rl_env_steps": 964840, "round": 121, "seed_start": 960000, "sr": 0.545, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 960000, "update_index": 363000}
50
+ {"chunk_transitions": 123000, "episodes": 200, "eval_index": 49, "gain": 0.025500000000000078, "gradient_updates": 369000, "ma": 0.5455, "n_success": 95, "policy_version": 123, "rl_env_steps": 980787, "round": 123, "seed_start": 960000, "sr": 0.475, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 980000, "update_index": 369000}
51
+ {"chunk_transitions": 126000, "episodes": 200, "eval_index": 50, "gain": 0.01200000000000001, "gradient_updates": 378000, "ma": 0.5405, "n_success": 106, "policy_version": 126, "rl_env_steps": 1004729, "round": 126, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1000000, "update_index": 378000}
52
+ {"chunk_transitions": 128000, "episodes": 200, "eval_index": 51, "gain": -0.0050000000000000044, "gradient_updates": 384000, "ma": 0.532, "n_success": 108, "policy_version": 128, "rl_env_steps": 1020661, "round": 128, "seed_start": 960000, "sr": 0.54, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1020000, "update_index": 384000}
53
+ {"chunk_transitions": 131000, "episodes": 200, "eval_index": 52, "gain": 0.000500000000000056, "gradient_updates": 393000, "ma": 0.5365, "n_success": 105, "policy_version": 131, "rl_env_steps": 1044581, "round": 131, "seed_start": 960000, "sr": 0.525, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1040000, "update_index": 393000}
54
+ {"chunk_transitions": 133000, "episodes": 200, "eval_index": 53, "gain": -0.009500000000000064, "gradient_updates": 399000, "ma": 0.5285, "n_success": 95, "policy_version": 133, "rl_env_steps": 1060525, "round": 133, "seed_start": 960000, "sr": 0.475, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1060000, "update_index": 399000}
55
+ {"chunk_transitions": 136000, "episodes": 200, "eval_index": 54, "gain": -0.015000000000000124, "gradient_updates": 408000, "ma": 0.5245, "n_success": 107, "policy_version": 136, "rl_env_steps": 1084427, "round": 136, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1080000, "update_index": 408000}
56
+ {"chunk_transitions": 138000, "episodes": 200, "eval_index": 55, "gain": -0.007000000000000006, "gradient_updates": 414000, "ma": 0.5295, "n_success": 112, "policy_version": 138, "rl_env_steps": 1100393, "round": 138, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1100000, "update_index": 414000}
57
+ {"chunk_transitions": 141000, "episodes": 200, "eval_index": 56, "gain": -0.0024999999999997247, "gradient_updates": 423000, "ma": 0.5360000000000001, "n_success": 123, "policy_version": 141, "rl_env_steps": 1124301, "round": 141, "seed_start": 960000, "sr": 0.615, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1120000, "update_index": 423000}
58
+ {"chunk_transitions": 143000, "episodes": 200, "eval_index": 57, "gain": -0.012499999999999956, "gradient_updates": 429000, "ma": 0.5345000000000001, "n_success": 109, "policy_version": 143, "rl_env_steps": 1140253, "round": 143, "seed_start": 960000, "sr": 0.545, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1140000, "update_index": 429000}
59
+ {"chunk_transitions": 146000, "episodes": 200, "eval_index": 58, "gain": -0.030500000000000083, "gradient_updates": 438000, "ma": 0.5235, "n_success": 87, "policy_version": 146, "rl_env_steps": 1164166, "round": 146, "seed_start": 960000, "sr": 0.435, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1160000, "update_index": 438000}
60
+ {"chunk_transitions": 148000, "episodes": 200, "eval_index": 59, "gain": -0.013999999999999901, "gradient_updates": 444000, "ma": 0.5315000000000001, "n_success": 111, "policy_version": 148, "rl_env_steps": 1180098, "round": 148, "seed_start": 960000, "sr": 0.555, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1180000, "update_index": 444000}
61
+ {"chunk_transitions": 151000, "episodes": 200, "eval_index": 60, "gain": -0.010000000000000009, "gradient_updates": 453000, "ma": 0.5305, "n_success": 104, "policy_version": 151, "rl_env_steps": 1203983, "round": 151, "seed_start": 960000, "sr": 0.52, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1200000, "update_index": 453000}
62
+ {"chunk_transitions": 154000, "episodes": 200, "eval_index": 61, "gain": 0.0035000000000000586, "gradient_updates": 462000, "ma": 0.5355000000000001, "n_success": 118, "policy_version": 154, "rl_env_steps": 1227895, "round": 154, "seed_start": 960000, "sr": 0.59, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1220000, "update_index": 462000}
63
+ {"chunk_transitions": 156000, "episodes": 200, "eval_index": 62, "gain": -0.007000000000000006, "gradient_updates": 468000, "ma": 0.5295, "n_success": 93, "policy_version": 156, "rl_env_steps": 1243838, "round": 156, "seed_start": 960000, "sr": 0.465, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1240000, "update_index": 468000}
64
+ {"chunk_transitions": 159000, "episodes": 200, "eval_index": 63, "gain": 0.007500000000000062, "gradient_updates": 477000, "ma": 0.536, "n_success": 108, "policy_version": 159, "rl_env_steps": 1267755, "round": 159, "seed_start": 960000, "sr": 0.54, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1260000, "update_index": 477000}
65
+ {"chunk_transitions": 161000, "episodes": 200, "eval_index": 64, "gain": 0.007500000000000062, "gradient_updates": 483000, "ma": 0.532, "n_success": 99, "policy_version": 161, "rl_env_steps": 1283705, "round": 161, "seed_start": 960000, "sr": 0.495, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1280000, "update_index": 483000}
66
+ {"chunk_transitions": 164000, "episodes": 200, "eval_index": 65, "gain": 0.0025000000000000577, "gradient_updates": 492000, "ma": 0.532, "n_success": 112, "policy_version": 164, "rl_env_steps": 1307623, "round": 164, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1300000, "update_index": 492000}
67
+ {"chunk_transitions": 166000, "episodes": 200, "eval_index": 66, "gain": -0.013500000000000179, "gradient_updates": 498000, "ma": 0.5225, "n_success": 104, "policy_version": 166, "rl_env_steps": 1323547, "round": 166, "seed_start": 960000, "sr": 0.52, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1320000, "update_index": 498000}
68
+ {"chunk_transitions": 169000, "episodes": 200, "eval_index": 67, "gain": -0.008500000000000063, "gradient_updates": 507000, "ma": 0.526, "n_success": 116, "policy_version": 169, "rl_env_steps": 1347470, "round": 169, "seed_start": 960000, "sr": 0.58, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1340000, "update_index": 507000}
69
+ {"chunk_transitions": 171000, "episodes": 200, "eval_index": 68, "gain": 0.012500000000000067, "gradient_updates": 513000, "ma": 0.536, "n_success": 107, "policy_version": 171, "rl_env_steps": 1363417, "round": 171, "seed_start": 960000, "sr": 0.535, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1360000, "update_index": 513000}
70
+ {"chunk_transitions": 174000, "episodes": 200, "eval_index": 69, "gain": 0.010000000000000009, "gradient_updates": 522000, "ma": 0.5415000000000001, "n_success": 122, "policy_version": 174, "rl_env_steps": 1387336, "round": 174, "seed_start": 960000, "sr": 0.61, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1380000, "update_index": 522000}
71
+ {"chunk_transitions": 176000, "episodes": 200, "eval_index": 70, "gain": 0.012000000000000122, "gradient_updates": 528000, "ma": 0.5425000000000001, "n_success": 106, "policy_version": 176, "rl_env_steps": 1403261, "round": 176, "seed_start": 960000, "sr": 0.53, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1400000, "update_index": 528000}
72
+ {"chunk_transitions": 179000, "episodes": 200, "eval_index": 71, "gain": 0.008000000000000007, "gradient_updates": 537000, "ma": 0.5435000000000001, "n_success": 120, "policy_version": 179, "rl_env_steps": 1427155, "round": 179, "seed_start": 960000, "sr": 0.6, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1420000, "update_index": 537000}
73
+ {"chunk_transitions": 181000, "episodes": 200, "eval_index": 72, "gain": 0.02300000000000002, "gradient_updates": 543000, "ma": 0.5525, "n_success": 111, "policy_version": 181, "rl_env_steps": 1443113, "round": 181, "seed_start": 960000, "sr": 0.555, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1440000, "update_index": 543000}
74
+ {"chunk_transitions": 184000, "episodes": 200, "eval_index": 73, "gain": 0.01749999999999985, "gradient_updates": 552000, "ma": 0.5534999999999999, "n_success": 110, "policy_version": 184, "rl_env_steps": 1467047, "round": 184, "seed_start": 960000, "sr": 0.55, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1460000, "update_index": 552000}
75
+ {"chunk_transitions": 186000, "episodes": 200, "eval_index": 74, "gain": 0.02849999999999997, "gradient_updates": 558000, "ma": 0.5605, "n_success": 113, "policy_version": 186, "rl_env_steps": 1482993, "round": 186, "seed_start": 960000, "sr": 0.565, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1480000, "update_index": 558000}
76
+ {"chunk_transitions": 189000, "episodes": 200, "eval_index": 75, "gain": 0.02749999999999997, "gradient_updates": 567000, "ma": 0.5595, "n_success": 110, "policy_version": 189, "rl_env_steps": 1506902, "round": 189, "seed_start": 960000, "sr": 0.55, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1500000, "update_index": 567000}
77
+ {"chunk_transitions": 191000, "episodes": 200, "eval_index": 76, "gain": 0.03600000000000003, "gradient_updates": 573000, "ma": 0.5585, "n_success": 102, "policy_version": 191, "rl_env_steps": 1522866, "round": 191, "seed_start": 960000, "sr": 0.51, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1520000, "update_index": 573000}
78
+ {"chunk_transitions": 194000, "episodes": 200, "eval_index": 77, "gain": 0.03049999999999997, "gradient_updates": 582000, "ma": 0.5565, "n_success": 112, "policy_version": 194, "rl_env_steps": 1546745, "round": 194, "seed_start": 960000, "sr": 0.56, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1540000, "update_index": 582000}
79
+ {"chunk_transitions": 196000, "episodes": 200, "eval_index": 78, "gain": 0.02199999999999991, "gradient_updates": 588000, "ma": 0.5579999999999999, "n_success": 110, "policy_version": 196, "rl_env_steps": 1562702, "round": 196, "seed_start": 960000, "sr": 0.55, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1560000, "update_index": 588000}
80
+ {"chunk_transitions": 199000, "episodes": 200, "eval_index": 79, "gain": 0.005499999999999838, "gradient_updates": 597000, "ma": 0.5469999999999999, "n_success": 100, "policy_version": 199, "rl_env_steps": 1586613, "round": 199, "seed_start": 960000, "sr": 0.5, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1580000, "update_index": 597000}
81
+ {"chunk_transitions": 201000, "episodes": 200, "eval_index": 80, "gain": 0.005499999999999838, "gradient_updates": 603000, "ma": 0.5479999999999999, "n_success": 108, "policy_version": 201, "rl_env_steps": 1602575, "round": 201, "seed_start": 960000, "sr": 0.54, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1600000, "update_index": 603000}
RL/Transport50/QC/diverse50/seed_2/eval/final.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b25379844a840f7d57b232050ca341ae3c5b8c81c66394f04dc4e37f5ec9b6e
3
+ size 6967
RL/Transport50/QC/diverse50/seed_2/final/source_metadata.json ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_sha256": "e136b372e5d08e8a0dc41d409398c01a57982fde2ece53aa12553e59923cc764",
3
+ "evaluation": {
4
+ "episodes": 300,
5
+ "imputed": false,
6
+ "n_success": 156,
7
+ "observed": true,
8
+ "seed_start": 800000,
9
+ "sr": 0.52,
10
+ "successes": [
11
+ 1,
12
+ 1,
13
+ 0,
14
+ 0,
15
+ 1,
16
+ 0,
17
+ 1,
18
+ 0,
19
+ 0,
20
+ 1,
21
+ 1,
22
+ 0,
23
+ 1,
24
+ 0,
25
+ 1,
26
+ 1,
27
+ 0,
28
+ 0,
29
+ 0,
30
+ 1,
31
+ 1,
32
+ 0,
33
+ 1,
34
+ 1,
35
+ 1,
36
+ 1,
37
+ 0,
38
+ 0,
39
+ 0,
40
+ 0,
41
+ 0,
42
+ 0,
43
+ 1,
44
+ 0,
45
+ 1,
46
+ 1,
47
+ 0,
48
+ 0,
49
+ 1,
50
+ 1,
51
+ 1,
52
+ 1,
53
+ 0,
54
+ 1,
55
+ 0,
56
+ 1,
57
+ 1,
58
+ 0,
59
+ 1,
60
+ 1,
61
+ 1,
62
+ 0,
63
+ 1,
64
+ 0,
65
+ 1,
66
+ 1,
67
+ 0,
68
+ 1,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 1,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 1,
78
+ 0,
79
+ 1,
80
+ 0,
81
+ 1,
82
+ 0,
83
+ 0,
84
+ 0,
85
+ 1,
86
+ 1,
87
+ 1,
88
+ 1,
89
+ 0,
90
+ 1,
91
+ 0,
92
+ 1,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 1,
98
+ 0,
99
+ 0,
100
+ 0,
101
+ 1,
102
+ 0,
103
+ 1,
104
+ 1,
105
+ 1,
106
+ 0,
107
+ 0,
108
+ 0,
109
+ 1,
110
+ 1,
111
+ 0,
112
+ 0,
113
+ 0,
114
+ 1,
115
+ 1,
116
+ 1,
117
+ 0,
118
+ 1,
119
+ 0,
120
+ 0,
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 0,
127
+ 0,
128
+ 1,
129
+ 0,
130
+ 1,
131
+ 1,
132
+ 0,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 0,
146
+ 0,
147
+ 1,
148
+ 1,
149
+ 0,
150
+ 1,
151
+ 1,
152
+ 1,
153
+ 1,
154
+ 1,
155
+ 0,
156
+ 0,
157
+ 1,
158
+ 1,
159
+ 0,
160
+ 1,
161
+ 0,
162
+ 1,
163
+ 0,
164
+ 0,
165
+ 0,
166
+ 0,
167
+ 0,
168
+ 0,
169
+ 1,
170
+ 1,
171
+ 0,
172
+ 1,
173
+ 1,
174
+ 1,
175
+ 0,
176
+ 1,
177
+ 0,
178
+ 1,
179
+ 0,
180
+ 1,
181
+ 1,
182
+ 0,
183
+ 1,
184
+ 0,
185
+ 0,
186
+ 0,
187
+ 1,
188
+ 1,
189
+ 1,
190
+ 0,
191
+ 1,
192
+ 1,
193
+ 0,
194
+ 1,
195
+ 0,
196
+ 1,
197
+ 1,
198
+ 1,
199
+ 0,
200
+ 1,
201
+ 1,
202
+ 1,
203
+ 0,
204
+ 1,
205
+ 1,
206
+ 0,
207
+ 0,
208
+ 0,
209
+ 1,
210
+ 0,
211
+ 0,
212
+ 0,
213
+ 0,
214
+ 0,
215
+ 0,
216
+ 0,
217
+ 0,
218
+ 0,
219
+ 1,
220
+ 0,
221
+ 1,
222
+ 1,
223
+ 0,
224
+ 1,
225
+ 1,
226
+ 1,
227
+ 1,
228
+ 1,
229
+ 0,
230
+ 1,
231
+ 1,
232
+ 0,
233
+ 1,
234
+ 0,
235
+ 0,
236
+ 1,
237
+ 0,
238
+ 1,
239
+ 0,
240
+ 1,
241
+ 0,
242
+ 0,
243
+ 0,
244
+ 1,
245
+ 1,
246
+ 0,
247
+ 0,
248
+ 0,
249
+ 0,
250
+ 0,
251
+ 1,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 1,
256
+ 0,
257
+ 0,
258
+ 1,
259
+ 1,
260
+ 0,
261
+ 1,
262
+ 1,
263
+ 0,
264
+ 1,
265
+ 0,
266
+ 1,
267
+ 1,
268
+ 1,
269
+ 0,
270
+ 1,
271
+ 1,
272
+ 0,
273
+ 0,
274
+ 1,
275
+ 0,
276
+ 1,
277
+ 0,
278
+ 0,
279
+ 0,
280
+ 1,
281
+ 0,
282
+ 1,
283
+ 1,
284
+ 1,
285
+ 0,
286
+ 0,
287
+ 1,
288
+ 1,
289
+ 1,
290
+ 1,
291
+ 1,
292
+ 1,
293
+ 0,
294
+ 0,
295
+ 0,
296
+ 0,
297
+ 0,
298
+ 0,
299
+ 0,
300
+ 0,
301
+ 1,
302
+ 1,
303
+ 0,
304
+ 1,
305
+ 1,
306
+ 0,
307
+ 0,
308
+ 1,
309
+ 1,
310
+ 1
311
+ ]
312
+ },
313
+ "status": "budget_complete"
314
+ }
RL/Transport50/QC/diverse50/seed_2/final/source_resolved_config.json ADDED
@@ -0,0 +1,1127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[0930-S2DEC]",
14
+ "key": "learner.qc.agent.act_bounds_json",
15
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
16
+ "upstream": null,
17
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}"
18
+ },
19
+ {
20
+ "decision_ref": "[0930-S2DEC]",
21
+ "key": "learner.qc.agent.act_max",
22
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
23
+ "upstream": 1.0,
24
+ "value": 4.179931163787842
25
+ },
26
+ {
27
+ "decision_ref": "[0930-S2DEC]",
28
+ "key": "learner.qc.agent.act_min",
29
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
30
+ "upstream": -1.0,
31
+ "value": -3.978297472000122
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.actor_lr_partition",
36
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
37
+ "upstream": "lr=0.0003 shared by actor and critic",
38
+ "value": 0.0001
39
+ },
40
+ {
41
+ "decision_ref": "[1002-S2ALIGN]",
42
+ "key": "learner.qc.agent.actor_num_samples",
43
+ "reason": "single BoN knob including QC TD",
44
+ "upstream": 32,
45
+ "value": 16
46
+ },
47
+ {
48
+ "decision_ref": "[1002-S2ALIGN]",
49
+ "key": "learner.qc.agent.best_of_n",
50
+ "reason": "single BoN knob: collection/eval/QC TD",
51
+ "upstream": 8,
52
+ "value": 16
53
+ },
54
+ {
55
+ "decision_ref": "docs/scope.md [0928-I]",
56
+ "key": "learner.qc.agent.bon_agg",
57
+ "reason": "Best-of-N selection by the ensemble mean",
58
+ "upstream": null,
59
+ "value": "mean"
60
+ },
61
+ {
62
+ "decision_ref": "docs/scope.md [0928-I]",
63
+ "key": "learner.qc.agent.bon_sft",
64
+ "reason": "variant qc flag bon_sft",
65
+ "upstream": "upstream_yaml",
66
+ "value": false
67
+ },
68
+ {
69
+ "decision_ref": "[0930-S2DEC]",
70
+ "key": "learner.qc.agent.critic_lr_partition",
71
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
72
+ "upstream": "lr=0.0003 shared by actor and critic",
73
+ "value": 0.0003
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.partition_optimizer",
78
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
79
+ "upstream": null,
80
+ "value": true
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.q_target_agg",
85
+ "reason": "REDQ-style TD target: min over a random critic subset",
86
+ "upstream": null,
87
+ "value": "subset_min"
88
+ },
89
+ {
90
+ "decision_ref": "docs/scope.md [0928-I]",
91
+ "key": "learner.qc.agent.q_target_replace",
92
+ "reason": "subset drawn without replacement",
93
+ "upstream": null,
94
+ "value": false
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.q_target_subset",
99
+ "reason": "TD-target subset size",
100
+ "upstream": null,
101
+ "value": 2
102
+ },
103
+ {
104
+ "decision_ref": "[0930-S2DEC]",
105
+ "key": "learner.qc.agent.value_hidden_dims",
106
+ "reason": "unified.critic_hidden (critics only; actors native)",
107
+ "upstream": [
108
+ 512,
109
+ 512,
110
+ 512,
111
+ 512
112
+ ],
113
+ "value": [
114
+ 512,
115
+ 512,
116
+ 512
117
+ ]
118
+ },
119
+ {
120
+ "decision_ref": "docs/scope.md [0928-I]",
121
+ "key": "learner.qc.training.bc_pi_steps",
122
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
123
+ "upstream": 1000000,
124
+ "value": 0
125
+ },
126
+ {
127
+ "decision_ref": "docs/scope.md [0928-I]",
128
+ "key": "learner.qc.loop.buffer.utd_online",
129
+ "reason": "UTD=3 per executed action chunk",
130
+ "upstream": 1,
131
+ "value": "3 gradient updates per executed chunk"
132
+ },
133
+ {
134
+ "decision_ref": "docs/scope.md [0928-I]",
135
+ "key": "learner.qc.loop.n_step",
136
+ "reason": "n-step=3 counted in action chunks",
137
+ "upstream": "1 chunk (sequence_length = horizon_length)",
138
+ "value": "3 executed chunks (unified_env_step window)"
139
+ },
140
+ {
141
+ "decision_ref": "docs/scope.md [0928-I]",
142
+ "key": "learner.qc.loop.critic_actor_ratio",
143
+ "reason": "critic every update, actor every 2nd global update",
144
+ "upstream": "1:1 (utd_q = utd_pi)",
145
+ "value": "2:1"
146
+ },
147
+ {
148
+ "decision_ref": "docs/scope.md [0928-I]",
149
+ "key": "learner.qc.loop.buffer.start_training",
150
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
151
+ "upstream": 10000,
152
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
153
+ },
154
+ {
155
+ "decision_ref": "docs/scope.md [0928-I]",
156
+ "key": "learner.qc.loop.buffer.offline_ratio",
157
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
158
+ "upstream": 0.0,
159
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
160
+ },
161
+ {
162
+ "decision_ref": "docs/scope.md [0928-I]",
163
+ "key": "learner.qc.loop.buffer.size",
164
+ "reason": "one shared replay of executed chunks; success buffer = mask",
165
+ "upstream": 2000000,
166
+ "value": "1000000 executed chunks, shared FIFO"
167
+ },
168
+ {
169
+ "decision_ref": "docs/scope.md [0928-I]",
170
+ "key": "learner.qc.actor_architecture",
171
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
172
+ "upstream": [
173
+ 512,
174
+ 512,
175
+ 512,
176
+ 512
177
+ ],
178
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
179
+ },
180
+ {
181
+ "decision_ref": "docs/scope.md [0928-I]",
182
+ "key": "learner.qc.observation",
183
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
184
+ "upstream": "low-dim state",
185
+ "value": "frozen Ri features c [146] from the checkpoint encoder"
186
+ },
187
+ {
188
+ "decision_ref": "[0930-S2DEC]",
189
+ "key": "learner.qc.reward",
190
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
191
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
192
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
193
+ },
194
+ {
195
+ "decision_ref": "[0930-S2DEC]",
196
+ "key": "learner.qc.td_target",
197
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
198
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
199
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
200
+ },
201
+ {
202
+ "decision_ref": "[0930-S2DEC]",
203
+ "key": "learner.qc.eval",
204
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
205
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
206
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
207
+ },
208
+ {
209
+ "decision_ref": "[1002-S2ALIGN]",
210
+ "key": "learner.qc.lr_schedule",
211
+ "reason": "RL env-step clock shared with extra success TD",
212
+ "upstream": "native per-optimizer schedule",
213
+ "value": {
214
+ "actor": 0.0001,
215
+ "critic": 0.0003,
216
+ "decay_end_env_steps": 1606196,
217
+ "min_ratio": 0.1,
218
+ "schedule": "warmup_cosine",
219
+ "temperature": 0.0003,
220
+ "warmup_env_steps": 20000
221
+ }
222
+ },
223
+ {
224
+ "decision_ref": "[0930-S2DEC]",
225
+ "key": "learner.qc.agent.q_min_q_max",
226
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
227
+ "upstream": [
228
+ -999.9999999999991,
229
+ 0.0
230
+ ],
231
+ "value": [
232
+ 0.0,
233
+ 1.0
234
+ ]
235
+ }
236
+ ],
237
+ "protocol": {
238
+ "checkpoint_every_rounds": 5,
239
+ "checkpoint_retention": {
240
+ "best": true,
241
+ "collapse_drop": {
242
+ "min_abs": 0.2,
243
+ "ratio": 0.5
244
+ },
245
+ "diagnostics": true,
246
+ "every_evals": 3
247
+ },
248
+ "collection": {
249
+ "n_envs": 4,
250
+ "train_seed_base": 100000000,
251
+ "train_seed_stride": 1000000,
252
+ "training_freq_chunks": 1000
253
+ },
254
+ "convergence": {
255
+ "early_stop": false,
256
+ "gain_lag": 10,
257
+ "high_sr": 0.95,
258
+ "high_sr_evals": 5,
259
+ "ma_window": 10,
260
+ "max_env_steps": 1606196,
261
+ "min_env_steps": 200000,
262
+ "min_gain": 0.02,
263
+ "zero_success_evals": 8
264
+ },
265
+ "critic_ensemble": {
266
+ "num": 10,
267
+ "policy": "mean",
268
+ "target": {
269
+ "kind": "subset_min",
270
+ "replace": false,
271
+ "size": 2
272
+ }
273
+ },
274
+ "eval": {
275
+ "best_of_n": 16,
276
+ "episodes": 200,
277
+ "every_env_steps": 20000,
278
+ "include_step0": true,
279
+ "seed_start": 960000,
280
+ "workers": 4
281
+ },
282
+ "final_eval": {
283
+ "episodes": 300,
284
+ "on_zero_stop": true,
285
+ "seed_start": 800000
286
+ },
287
+ "n_step_chunks": 3,
288
+ "offline": {
289
+ "adaptive_ratio": false,
290
+ "cohort_size": 50,
291
+ "episodes": 50,
292
+ "learners_without_offline": [
293
+ "dsrl_sac"
294
+ ],
295
+ "ratio": {
296
+ "decay_env_steps": {
297
+ "can": 160000,
298
+ "square": 320000,
299
+ "tool_hang": 640000,
300
+ "transport": 320000
301
+ },
302
+ "end": 0.1,
303
+ "start": {
304
+ "can": 0.7,
305
+ "square": 0.5,
306
+ "tool_hang": 0.5,
307
+ "transport": 0.9
308
+ }
309
+ },
310
+ "selection": "ascending_demo_index",
311
+ "source": "bc_regime",
312
+ "unsupported_learners": [
313
+ "zprl"
314
+ ]
315
+ },
316
+ "replay": {
317
+ "capacity_chunks": 1000000,
318
+ "pin_start_successes": false,
319
+ "success_buffer": "success_mask_on_shared_fifo",
320
+ "success_skip_rule": "skip_below_success_batch_size",
321
+ "unit": "executed_chunk"
322
+ },
323
+ "start": {
324
+ "max_episodes": 200,
325
+ "mode": "offline_buffer",
326
+ "policy": {
327
+ "default": "base_cfm_ode",
328
+ "zprl": "vib_base"
329
+ },
330
+ "target_successes": 20
331
+ },
332
+ "unified": {
333
+ "actor_target_tau": 0.005,
334
+ "critic_hidden": [
335
+ 512,
336
+ 512,
337
+ 512
338
+ ],
339
+ "critic_target_every_critic_step": true,
340
+ "critic_target_tau": 0.005,
341
+ "dsrl_noise_imitation_reference_dims": 28,
342
+ "gamma_env": 0.999,
343
+ "lr": {
344
+ "actor": 0.0001,
345
+ "critic": 0.0003,
346
+ "decay_end_env_steps": 1606196,
347
+ "min_ratio": 0.1,
348
+ "schedule": "warmup_cosine",
349
+ "temperature": 0.0003,
350
+ "warmup_env_steps": 20000
351
+ },
352
+ "reward": "binary_first_success",
353
+ "success_td_polyak": true,
354
+ "timeout_bootstrap": false,
355
+ "zprl_imitation": "mse_pretanh_mean"
356
+ },
357
+ "updates": {
358
+ "batch_size": 256,
359
+ "critic_per_actor": 2,
360
+ "success_batch_size": 256,
361
+ "utd_per_chunk": 3
362
+ },
363
+ "version": "study2_protocol_v10"
364
+ },
365
+ "protocol_hash": "bdfdfb18fc67ba88af67e02b6592b190b509c5ad68cbd7357fcbca9fa8c796b9",
366
+ "provenance_extra": {
367
+ "agent_library": {
368
+ "framework": "jax",
369
+ "sources": {
370
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
371
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
372
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
373
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
374
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
375
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
376
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
377
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
378
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
379
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
380
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
381
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
382
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
383
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
384
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
385
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
386
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
387
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
388
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
389
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
390
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
391
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
392
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
393
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
394
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
395
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
396
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
397
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
398
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
399
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
400
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
401
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
402
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
403
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
404
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
405
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
406
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
407
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
408
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
409
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
410
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
411
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
412
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
413
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
414
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
415
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
416
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
417
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
418
+ "train.py": "d1ec607894c8e126e710b5a4ef6c48a4e9bf6c66c0872ce2e2bdc1c65b66d98d",
419
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
420
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
421
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
422
+ },
423
+ "version": "study2_jax_agents_v2"
424
+ },
425
+ "base_framework": "jax",
426
+ "bc_identity": {
427
+ "architecture": "dice_vision_resnet_flow_unet_v1",
428
+ "family": "StandardBaseBC",
429
+ "framework": "jax"
430
+ },
431
+ "checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261022_transport_focused50_v10_four/base/diverse50/epoch_3000.pkl",
432
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
433
+ "experiment": "transport_8",
434
+ "frozen_hash": "2414629d8dcbea1d75ca2831d5e0b649e65cffd56aa0ed22212fd0c3ec60b1bc",
435
+ "learner": {
436
+ "action_bounds": {
437
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
438
+ "compat_act_max": 4.179931163787842,
439
+ "compat_act_min": -3.978297472000122,
440
+ "hi": [
441
+ 1.0,
442
+ 1.0,
443
+ 1.0,
444
+ 1.0,
445
+ 2.775709390640259,
446
+ 2.2864010334014893,
447
+ 1.0,
448
+ 1.0,
449
+ 1.0,
450
+ 1.0,
451
+ 1.4753823280334473,
452
+ 4.088724613189697,
453
+ 4.179931163787842,
454
+ 1.0
455
+ ],
456
+ "kind": "per_dimension",
457
+ "lo": [
458
+ -1.0,
459
+ -1.0,
460
+ -1.0,
461
+ -1.4882197380065918,
462
+ -3.233746290206909,
463
+ -2.5405781269073486,
464
+ -1.0,
465
+ -1.0,
466
+ -1.0,
467
+ -1.0,
468
+ -1.7870254516601562,
469
+ -3.208747148513794,
470
+ -3.978297472000122,
471
+ -1.0
472
+ ],
473
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
474
+ },
475
+ "agent_config": {
476
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
477
+ "act_max": 4.179931163787842,
478
+ "act_min": -3.978297472000122,
479
+ "action_chunking": true,
480
+ "action_critic_hidden_dims": null,
481
+ "action_dim": 14,
482
+ "action_num_qs": 10,
483
+ "action_repeat": false,
484
+ "actor_backbone": "mlp",
485
+ "actor_decay_steps": 50000,
486
+ "actor_end_value": 2e-05,
487
+ "actor_hidden_dims": [
488
+ 512,
489
+ 512,
490
+ 512,
491
+ 512
492
+ ],
493
+ "actor_layer_norm": false,
494
+ "actor_lr": 0.0003,
495
+ "actor_lr_partition": 0.0001,
496
+ "actor_num_samples": 16,
497
+ "actor_obs": "state",
498
+ "actor_scheduler": "cosine",
499
+ "actor_tau": 0.005,
500
+ "actor_type": "best-of-n",
501
+ "actor_warmup_steps": 2000,
502
+ "actor_weight_decay": 0.0,
503
+ "adv_clip_min": null,
504
+ "agent_name": "acfql",
505
+ "alpha": 500,
506
+ "batch_size": 256,
507
+ "best_of_n": 16,
508
+ "bon_agg": "mean",
509
+ "bon_sft": false,
510
+ "bon_sft_coeff": 1.0,
511
+ "bootstrap_ratio": 0.5,
512
+ "bptt_num_samples": 4,
513
+ "calql_q_subsample": 2,
514
+ "clip_grad_norm": 1000.0,
515
+ "clip_intermediate_actions": true,
516
+ "clip_max_epsilon_multiplier": 1.0,
517
+ "clip_min_epsilon_multiplier": 1.0,
518
+ "constant_noise_std": 0.01,
519
+ "cql_alpha": 5.0,
520
+ "cql_n_actions": 10,
521
+ "cql_temp": 1.0,
522
+ "critic_backbone": "mlp",
523
+ "critic_decay_steps": 5000,
524
+ "critic_end_value": 1e-08,
525
+ "critic_loss_type": "mse",
526
+ "critic_lr": 0.0003,
527
+ "critic_lr_partition": 0.0003,
528
+ "critic_obs": "state",
529
+ "critic_scheduler": "constant",
530
+ "critic_warmup_steps": 500,
531
+ "critic_weight_decay": 1e-05,
532
+ "denoised_clip_value": 1.0,
533
+ "discount": 0.999,
534
+ "edit_action_scale": 0.5,
535
+ "edit_actor_hidden_dims": null,
536
+ "edit_hidden_dims": [
537
+ 256,
538
+ 256,
539
+ 256
540
+ ],
541
+ "edit_init_temperature": 1.0,
542
+ "edit_layer_norm": true,
543
+ "edit_lr": 0.0003,
544
+ "edit_policy_bound": 0.1,
545
+ "edit_start_step": 100000,
546
+ "edit_target_entropy": null,
547
+ "edit_weight_decay": 0.0,
548
+ "encoder": null,
549
+ "entropy_backup": false,
550
+ "flow_steps": 10,
551
+ "fql_distillation_coeff": 10.0,
552
+ "fql_train_main_policy": true,
553
+ "freeze_vision_encoder": false,
554
+ "ft_flow_steps": 10,
555
+ "horizon_length": 8,
556
+ "init_temperature": 1.0,
557
+ "layer_norm": true,
558
+ "lr": 0.0003,
559
+ "lr_action_critic": null,
560
+ "lr_actor": null,
561
+ "lr_actor_bc_flow": null,
562
+ "lr_critic": null,
563
+ "lr_edit_actor": null,
564
+ "lr_noise_critic": null,
565
+ "max_noise_std": 0.01,
566
+ "mc_regression": false,
567
+ "mc_regression_coeff": 1.0,
568
+ "min_noise_std": 0.01,
569
+ "mip_q_ensemble": false,
570
+ "mip_q_noise_scale": 1.0,
571
+ "mip_q_t_star": 0.9,
572
+ "mip_t_star": 0.999,
573
+ "muon_beta": 0.95,
574
+ "muon_nesterov": true,
575
+ "muon_ns_steps": 5,
576
+ "n_action_samples": 8,
577
+ "n_edit_samples": 8,
578
+ "noise_critic_grad_steps": 1,
579
+ "noise_dim": null,
580
+ "noise_net_hidden_dims": [
581
+ 256,
582
+ 256
583
+ ],
584
+ "noise_net_layer_norm": true,
585
+ "normalize_act_space_dimension": false,
586
+ "normalize_denoising_horizon": false,
587
+ "normalize_q_loss": false,
588
+ "num_bins": 256,
589
+ "num_ensemble_members": 10,
590
+ "num_qs": 10,
591
+ "ob_dims": [
592
+ 146
593
+ ],
594
+ "one_step_decay_steps": null,
595
+ "one_step_end_value": null,
596
+ "one_step_hidden_dims": [
597
+ 512,
598
+ 512,
599
+ 512,
600
+ 512
601
+ ],
602
+ "one_step_layer_norm": false,
603
+ "one_step_lr": null,
604
+ "one_step_scheduler": null,
605
+ "one_step_warmup_steps": null,
606
+ "optimizer_action_critic": null,
607
+ "optimizer_actor": "adam",
608
+ "optimizer_actor_bc_flow": "adam",
609
+ "optimizer_critic": "adam",
610
+ "optimizer_edit_actor": null,
611
+ "paligemma_tokenizer_path": null,
612
+ "partition_optimizer": true,
613
+ "policy_type": "flow",
614
+ "ppo_batch_size": 256,
615
+ "ppo_lr": 4.5e-05,
616
+ "proprio_dim": null,
617
+ "q_agg": "mean",
618
+ "q_max": 1.0,
619
+ "q_min": 0.0,
620
+ "q_target_agg": "subset_min",
621
+ "q_target_replace": false,
622
+ "q_target_subset": 2,
623
+ "q_variance_reduction": false,
624
+ "q_vr_num_samples": 8,
625
+ "q_vr_reduction": "mean",
626
+ "randn_clip_value": 3.0,
627
+ "shortcut_denoising_steps": 4,
628
+ "shortcut_inference_steps": 4,
629
+ "simba_actor_hidden_dims": [
630
+ 1024,
631
+ 1024
632
+ ],
633
+ "simba_critic_hidden_dims": [
634
+ 1024,
635
+ 1024,
636
+ 1024
637
+ ],
638
+ "subsample_bon": false,
639
+ "target_entropy": null,
640
+ "tau": 0.005,
641
+ "tf_pi_conditioning": "adaln",
642
+ "tf_pi_dropout": 0.0,
643
+ "tf_pi_embed_dim": 256,
644
+ "tf_pi_heads": 4,
645
+ "tf_pi_layers": 8,
646
+ "tf_q_conditioning": "adaln",
647
+ "tf_q_dropout": 0.0,
648
+ "tf_q_embed_dim": 256,
649
+ "tf_q_heads": 4,
650
+ "tf_q_layers": 8,
651
+ "time_embedding": "sinusoidal",
652
+ "time_embedding_dim": 32,
653
+ "train_one_step_in_offline": true,
654
+ "train_one_step_in_online": true,
655
+ "useFiLM": false,
656
+ "useSimBa": false,
657
+ "useTF": false,
658
+ "use_constant_noise": true,
659
+ "use_constant_scheduler_for_bc": false,
660
+ "use_denoiser": false,
661
+ "use_edit_actor": false,
662
+ "use_mip_q": false,
663
+ "use_muon": false,
664
+ "use_one_step_policy": false,
665
+ "use_shortcut": false,
666
+ "use_state": "proprio",
667
+ "use_success_buffer_q": false,
668
+ "value_hidden_dims": [
669
+ 512,
670
+ 512,
671
+ 512
672
+ ],
673
+ "vit_encoder": "minvit",
674
+ "vit_img_proj_dim": 0,
675
+ "vit_state_proj_dim": 0
676
+ },
677
+ "convention": "unified_env_step",
678
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
679
+ "gamma_env": 0.999,
680
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
681
+ "learner": "qc",
682
+ "optimizers": {
683
+ "actor_lr": 0.0001,
684
+ "critic_lr": 0.0003,
685
+ "reset_optimizers_with_lr": "not used",
686
+ "schedule": {
687
+ "actor": 0.0001,
688
+ "critic": 0.0003,
689
+ "decay_end_env_steps": 1606196,
690
+ "min_ratio": 0.1,
691
+ "schedule": "warmup_cosine",
692
+ "temperature": 0.0003,
693
+ "warmup_env_steps": 20000
694
+ }
695
+ },
696
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
697
+ "recipe": {
698
+ "derived": {
699
+ "agent.actor_num_samples": {
700
+ "after": 8,
701
+ "before": 32
702
+ },
703
+ "agent.actor_type": {
704
+ "after": "best-of-n",
705
+ "before": "upstream_yaml"
706
+ },
707
+ "agent.bon_sft": {
708
+ "after": false,
709
+ "before": "upstream_yaml"
710
+ }
711
+ },
712
+ "explicit_overrides": {},
713
+ "source": "third_party/ogpo/scripts/qc/transport.sh",
714
+ "upstream_exact": false
715
+ },
716
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
717
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
718
+ "scan_block": 100,
719
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
720
+ "success_terms": {
721
+ "bc_on_success_batch": false,
722
+ "critic_success_td": false,
723
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
724
+ },
725
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
726
+ "unified_notes": {
727
+ "critic_target_every_critic_step": {
728
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
729
+ "value": true
730
+ },
731
+ "success_td_polyak": {
732
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
733
+ "value": true
734
+ }
735
+ },
736
+ "upstream_config": {
737
+ "agent": {
738
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
739
+ "act_max": 4.179931163787842,
740
+ "act_min": -3.978297472000122,
741
+ "action_chunking": true,
742
+ "action_critic_hidden_dims": null,
743
+ "action_dim": null,
744
+ "action_num_qs": 10,
745
+ "action_repeat": false,
746
+ "actor_backbone": "mlp",
747
+ "actor_decay_steps": 50000,
748
+ "actor_end_value": 2e-05,
749
+ "actor_hidden_dims": [
750
+ 512,
751
+ 512,
752
+ 512,
753
+ 512
754
+ ],
755
+ "actor_layer_norm": false,
756
+ "actor_lr": 0.0003,
757
+ "actor_lr_partition": 0.0001,
758
+ "actor_num_samples": 16,
759
+ "actor_obs": "state",
760
+ "actor_scheduler": "cosine",
761
+ "actor_tau": 0.005,
762
+ "actor_type": "best-of-n",
763
+ "actor_warmup_steps": 2000,
764
+ "actor_weight_decay": 0.0,
765
+ "adv_clip_min": null,
766
+ "agent_name": "acfql",
767
+ "alpha": 500,
768
+ "batch_size": 256,
769
+ "best_of_n": 16,
770
+ "bon_agg": "mean",
771
+ "bon_sft": false,
772
+ "bon_sft_coeff": 1.0,
773
+ "bootstrap_ratio": 0.5,
774
+ "bptt_num_samples": 4,
775
+ "calql_q_subsample": 2,
776
+ "clip_grad_norm": 1000.0,
777
+ "clip_intermediate_actions": true,
778
+ "clip_max_epsilon_multiplier": 1.0,
779
+ "clip_min_epsilon_multiplier": 1.0,
780
+ "constant_noise_std": 0.01,
781
+ "cql_alpha": 5.0,
782
+ "cql_n_actions": 10,
783
+ "cql_temp": 1.0,
784
+ "critic_backbone": "mlp",
785
+ "critic_decay_steps": 5000,
786
+ "critic_end_value": 1e-08,
787
+ "critic_loss_type": "mse",
788
+ "critic_lr": 0.0003,
789
+ "critic_lr_partition": 0.0003,
790
+ "critic_obs": "state",
791
+ "critic_scheduler": "constant",
792
+ "critic_warmup_steps": 500,
793
+ "critic_weight_decay": 1e-05,
794
+ "denoised_clip_value": 1.0,
795
+ "discount": 0.999,
796
+ "edit_action_scale": 0.5,
797
+ "edit_actor_hidden_dims": null,
798
+ "edit_hidden_dims": [
799
+ 256,
800
+ 256,
801
+ 256
802
+ ],
803
+ "edit_init_temperature": 1.0,
804
+ "edit_layer_norm": true,
805
+ "edit_lr": 0.0003,
806
+ "edit_policy_bound": 0.1,
807
+ "edit_start_step": 100000,
808
+ "edit_target_entropy": null,
809
+ "edit_weight_decay": 0.0,
810
+ "encoder": null,
811
+ "entropy_backup": false,
812
+ "flow_steps": 10,
813
+ "fql_distillation_coeff": 10.0,
814
+ "fql_train_main_policy": true,
815
+ "freeze_vision_encoder": false,
816
+ "ft_flow_steps": 10,
817
+ "horizon_length": 8,
818
+ "init_temperature": 1.0,
819
+ "layer_norm": true,
820
+ "lr": 0.0003,
821
+ "lr_action_critic": null,
822
+ "lr_actor": null,
823
+ "lr_actor_bc_flow": null,
824
+ "lr_critic": null,
825
+ "lr_edit_actor": null,
826
+ "lr_noise_critic": null,
827
+ "max_noise_std": 0.01,
828
+ "mc_regression": false,
829
+ "mc_regression_coeff": 1.0,
830
+ "min_noise_std": 0.01,
831
+ "mip_q_ensemble": false,
832
+ "mip_q_noise_scale": 1.0,
833
+ "mip_q_t_star": 0.9,
834
+ "mip_t_star": 0.999,
835
+ "muon_beta": 0.95,
836
+ "muon_nesterov": true,
837
+ "muon_ns_steps": 5,
838
+ "n_action_samples": 8,
839
+ "n_edit_samples": 8,
840
+ "noise_critic_grad_steps": 1,
841
+ "noise_dim": null,
842
+ "noise_net_hidden_dims": [
843
+ 256,
844
+ 256
845
+ ],
846
+ "noise_net_layer_norm": true,
847
+ "normalize_act_space_dimension": false,
848
+ "normalize_denoising_horizon": false,
849
+ "normalize_q_loss": false,
850
+ "num_bins": 256,
851
+ "num_ensemble_members": 10,
852
+ "num_qs": 10,
853
+ "ob_dims": null,
854
+ "one_step_decay_steps": null,
855
+ "one_step_end_value": null,
856
+ "one_step_hidden_dims": [
857
+ 512,
858
+ 512,
859
+ 512,
860
+ 512
861
+ ],
862
+ "one_step_layer_norm": false,
863
+ "one_step_lr": null,
864
+ "one_step_scheduler": null,
865
+ "one_step_warmup_steps": null,
866
+ "optimizer_action_critic": null,
867
+ "optimizer_actor": "adam",
868
+ "optimizer_actor_bc_flow": "adam",
869
+ "optimizer_critic": "adam",
870
+ "optimizer_edit_actor": null,
871
+ "paligemma_tokenizer_path": null,
872
+ "partition_optimizer": true,
873
+ "policy_type": "flow",
874
+ "ppo_batch_size": 256,
875
+ "ppo_lr": 4.5e-05,
876
+ "proprio_dim": null,
877
+ "q_agg": "mean",
878
+ "q_max": null,
879
+ "q_min": null,
880
+ "q_target_agg": "subset_min",
881
+ "q_target_replace": false,
882
+ "q_target_subset": 2,
883
+ "q_variance_reduction": false,
884
+ "q_vr_num_samples": 8,
885
+ "q_vr_reduction": "mean",
886
+ "randn_clip_value": 3.0,
887
+ "shortcut_denoising_steps": 4,
888
+ "shortcut_inference_steps": 4,
889
+ "simba_actor_hidden_dims": [
890
+ 1024,
891
+ 1024
892
+ ],
893
+ "simba_critic_hidden_dims": [
894
+ 1024,
895
+ 1024,
896
+ 1024
897
+ ],
898
+ "subsample_bon": false,
899
+ "target_entropy": null,
900
+ "tau": 0.005,
901
+ "tf_pi_conditioning": "adaln",
902
+ "tf_pi_dropout": 0.0,
903
+ "tf_pi_embed_dim": 256,
904
+ "tf_pi_heads": 4,
905
+ "tf_pi_layers": 8,
906
+ "tf_q_conditioning": "adaln",
907
+ "tf_q_dropout": 0.0,
908
+ "tf_q_embed_dim": 256,
909
+ "tf_q_heads": 4,
910
+ "tf_q_layers": 8,
911
+ "time_embedding": "sinusoidal",
912
+ "time_embedding_dim": 32,
913
+ "train_one_step_in_offline": true,
914
+ "train_one_step_in_online": true,
915
+ "useFiLM": false,
916
+ "useSimBa": false,
917
+ "useTF": false,
918
+ "use_constant_noise": true,
919
+ "use_constant_scheduler_for_bc": false,
920
+ "use_denoiser": false,
921
+ "use_edit_actor": false,
922
+ "use_mip_q": false,
923
+ "use_muon": false,
924
+ "use_one_step_policy": false,
925
+ "use_shortcut": false,
926
+ "use_state": "proprio",
927
+ "use_success_buffer_q": false,
928
+ "value_hidden_dims": [
929
+ 512,
930
+ 512,
931
+ 512
932
+ ],
933
+ "vit_encoder": "minvit",
934
+ "vit_img_proj_dim": 0,
935
+ "vit_state_proj_dim": 0
936
+ },
937
+ "bc": {
938
+ "clip_bc": false,
939
+ "clip_bc_threshold": 0.45,
940
+ "clip_bc_wrt": "ODE"
941
+ },
942
+ "buffer": {
943
+ "offline_ratio": 0.0,
944
+ "size": 2000000,
945
+ "start_training": 10000,
946
+ "utd_online": 1,
947
+ "utd_pi": null,
948
+ "utd_q": null,
949
+ "utd_v": null,
950
+ "utd_warmup": 1
951
+ },
952
+ "checkpoint": {
953
+ "ep_resume": 0,
954
+ "restore_actor_path": null,
955
+ "restore_critic_path": null,
956
+ "restore_encoder_path": null
957
+ },
958
+ "dataset": {
959
+ "num_offline_trajs": -1,
960
+ "ogbench_dir": null,
961
+ "p_aug": 0.0,
962
+ "proportion": 1.0,
963
+ "replace_interval": 1000
964
+ },
965
+ "env": {
966
+ "name": "transport-mh-low_dim",
967
+ "post_success_steps": 0,
968
+ "rew_fn": "sparse",
969
+ "sparse": false
970
+ },
971
+ "eval": {
972
+ "eval_episodes": 2,
973
+ "n_eval_envs": 32,
974
+ "plot_q_vs_mc": false,
975
+ "video_episodes": 0,
976
+ "video_frame_skip": 3
977
+ },
978
+ "experiment": {
979
+ "log": true,
980
+ "project": "QC",
981
+ "run_group": "transport",
982
+ "run_name": "qc_transport",
983
+ "save_dir": "exp/ogpo",
984
+ "seed": 1,
985
+ "wandb_name": "qc_transport"
986
+ },
987
+ "intervals": {
988
+ "eval_interval": 20000,
989
+ "eval_interval_bc": 5000,
990
+ "log_interval": 5000,
991
+ "preemption_backup_interval": -1,
992
+ "save_interval": 500000,
993
+ "save_interval_bc": -1,
994
+ "step_data_log_interval": -1
995
+ },
996
+ "success_buffer": {
997
+ "batch_size": 256,
998
+ "use_success_buffer": false
999
+ },
1000
+ "training": {
1001
+ "bc_only": false,
1002
+ "bc_pi_steps": 0,
1003
+ "bc_q_steps": 0,
1004
+ "bc_refine_steps": 0,
1005
+ "calql_steps": 0,
1006
+ "force_jax_sync": true,
1007
+ "gpu_mode": "single",
1008
+ "online_steps": 6000000,
1009
+ "q_warmup_steps": 0
1010
+ }
1011
+ },
1012
+ "upstream_source": {
1013
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1014
+ "patches_sha256": {
1015
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1016
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1017
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1018
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1019
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1020
+ }
1021
+ }
1022
+ },
1023
+ "observation_mode": "pixel_state",
1024
+ "offline_buffer": {
1025
+ "actions_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1026
+ "aux_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1027
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
1028
+ "chunks": 3183,
1029
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1030
+ "decision_ref": "[1017-S2OFFBC]",
1031
+ "env_steps": 25283,
1032
+ "features_sha256": "9f82519c4cb371ffbbb2c3f5ae9208cb7dc6e48ca9e0ba28b7c3845dc1f1148f",
1033
+ "horizon": 8,
1034
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
1035
+ "learner_coordinates": "JaxStudy2Learner:qc",
1036
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1037
+ "ratio": {
1038
+ "adaptive_ratio": false,
1039
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1040
+ },
1041
+ "selection": {
1042
+ "cohort_size": 50,
1043
+ "demos": [
1044
+ "demo_0",
1045
+ "demo_7",
1046
+ "demo_8",
1047
+ "demo_10",
1048
+ "demo_11",
1049
+ "demo_13",
1050
+ "demo_14",
1051
+ "demo_17",
1052
+ "demo_20",
1053
+ "demo_22",
1054
+ "demo_28",
1055
+ "demo_29",
1056
+ "demo_33",
1057
+ "demo_38",
1058
+ "demo_39",
1059
+ "demo_48",
1060
+ "demo_49",
1061
+ "demo_53",
1062
+ "demo_59",
1063
+ "demo_61",
1064
+ "demo_69",
1065
+ "demo_75",
1066
+ "demo_76",
1067
+ "demo_81",
1068
+ "demo_83",
1069
+ "demo_88",
1070
+ "demo_95",
1071
+ "demo_96",
1072
+ "demo_102",
1073
+ "demo_103",
1074
+ "demo_105",
1075
+ "demo_108",
1076
+ "demo_119",
1077
+ "demo_120",
1078
+ "demo_126",
1079
+ "demo_128",
1080
+ "demo_139",
1081
+ "demo_151",
1082
+ "demo_152",
1083
+ "demo_154",
1084
+ "demo_159",
1085
+ "demo_163",
1086
+ "demo_165",
1087
+ "demo_171",
1088
+ "demo_173",
1089
+ "demo_177",
1090
+ "demo_180",
1091
+ "demo_181",
1092
+ "demo_182",
1093
+ "demo_190"
1094
+ ],
1095
+ "manifest": "/ssd/WorkSpace/RL-Postrain/data/bc/versions/ph_distribution_v2/manifests/transport.json",
1096
+ "manifest_sha256": "8f2569c5918df85bccd661f8b4c8f76fd73948ca0c4ff96487e949ddbdb4013f",
1097
+ "regime": "diverse50",
1098
+ "source": "bc_regime",
1099
+ "source_ph_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1100
+ "split": {
1101
+ "training": 45,
1102
+ "validation": 5
1103
+ },
1104
+ "task": "transport"
1105
+ }
1106
+ },
1107
+ "offline_role": "mixed",
1108
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1109
+ "smoke": false,
1110
+ "task": "transport",
1111
+ "task_spec_hash": "fd9c198a613abab85d95bda9521355ae6d926bcf7da1895ca7a1dd03325117cb"
1112
+ },
1113
+ "seed": 2,
1114
+ "start_policy": "base_cfm_ode",
1115
+ "variant": {
1116
+ "flags": {
1117
+ "bon_sft": false,
1118
+ "use_success_buffer": false
1119
+ },
1120
+ "learner": "qc",
1121
+ "name": "qc",
1122
+ "phase": 1,
1123
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1124
+ "version": "study2_variants_v4"
1125
+ },
1126
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1127
+ }
RL/Transport50/QC/diverse50/seed_2/final/state.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e136b372e5d08e8a0dc41d409398c01a57982fde2ece53aa12553e59923cc764
3
+ size 215002264
RL/Transport50/QC/diverse50/seed_2/frozen_base.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "after": "2414629d8dcbea1d75ca2831d5e0b649e65cffd56aa0ed22212fd0c3ec60b1bc",
3
+ "before": "2414629d8dcbea1d75ca2831d5e0b649e65cffd56aa0ed22212fd0c3ec60b1bc",
4
+ "checkpoint_unchanged": true,
5
+ "unchanged": true
6
+ }
RL/Transport50/QC/diverse50/seed_2/offline_buffer.json ADDED
@@ -0,0 +1,447 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "identity": {
3
+ "actions_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
4
+ "aux_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
5
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
6
+ "chunks": 3183,
7
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
8
+ "decision_ref": "[1017-S2OFFBC]",
9
+ "env_steps": 25283,
10
+ "features_sha256": "9f82519c4cb371ffbbb2c3f5ae9208cb7dc6e48ca9e0ba28b7c3845dc1f1148f",
11
+ "horizon": 8,
12
+ "learner_coordinates": "JaxStudy2Learner:qc",
13
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
14
+ "ratio": {
15
+ "adaptive_ratio": false,
16
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
17
+ },
18
+ "selection": {
19
+ "cohort_size": 50,
20
+ "demos": [
21
+ "demo_0",
22
+ "demo_7",
23
+ "demo_8",
24
+ "demo_10",
25
+ "demo_11",
26
+ "demo_13",
27
+ "demo_14",
28
+ "demo_17",
29
+ "demo_20",
30
+ "demo_22",
31
+ "demo_28",
32
+ "demo_29",
33
+ "demo_33",
34
+ "demo_38",
35
+ "demo_39",
36
+ "demo_48",
37
+ "demo_49",
38
+ "demo_53",
39
+ "demo_59",
40
+ "demo_61",
41
+ "demo_69",
42
+ "demo_75",
43
+ "demo_76",
44
+ "demo_81",
45
+ "demo_83",
46
+ "demo_88",
47
+ "demo_95",
48
+ "demo_96",
49
+ "demo_102",
50
+ "demo_103",
51
+ "demo_105",
52
+ "demo_108",
53
+ "demo_119",
54
+ "demo_120",
55
+ "demo_126",
56
+ "demo_128",
57
+ "demo_139",
58
+ "demo_151",
59
+ "demo_152",
60
+ "demo_154",
61
+ "demo_159",
62
+ "demo_163",
63
+ "demo_165",
64
+ "demo_171",
65
+ "demo_173",
66
+ "demo_177",
67
+ "demo_180",
68
+ "demo_181",
69
+ "demo_182",
70
+ "demo_190"
71
+ ],
72
+ "manifest": "/ssd/WorkSpace/RL-Postrain/data/bc/versions/ph_distribution_v2/manifests/transport.json",
73
+ "manifest_sha256": "8f2569c5918df85bccd661f8b4c8f76fd73948ca0c4ff96487e949ddbdb4013f",
74
+ "regime": "diverse50",
75
+ "source": "bc_regime",
76
+ "source_ph_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
77
+ "split": {
78
+ "training": 45,
79
+ "validation": 5
80
+ },
81
+ "task": "transport"
82
+ }
83
+ },
84
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
85
+ "stats": {
86
+ "chunks": 3183,
87
+ "dataset": "/ssd/WorkSpace/RL-Postrain/data/robomimic/transport/ph/low_dim_v15.hdf5",
88
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
89
+ "env_steps": 25283,
90
+ "per_demo": [
91
+ {
92
+ "chunks": 89,
93
+ "demo": "demo_0",
94
+ "env_steps": 710,
95
+ "frames": 714,
96
+ "terminal_next_frame": 710
97
+ },
98
+ {
99
+ "chunks": 59,
100
+ "demo": "demo_7",
101
+ "env_steps": 471,
102
+ "frames": 475,
103
+ "terminal_next_frame": 471
104
+ },
105
+ {
106
+ "chunks": 59,
107
+ "demo": "demo_8",
108
+ "env_steps": 471,
109
+ "frames": 475,
110
+ "terminal_next_frame": 471
111
+ },
112
+ {
113
+ "chunks": 77,
114
+ "demo": "demo_10",
115
+ "env_steps": 612,
116
+ "frames": 616,
117
+ "terminal_next_frame": 612
118
+ },
119
+ {
120
+ "chunks": 62,
121
+ "demo": "demo_11",
122
+ "env_steps": 493,
123
+ "frames": 497,
124
+ "terminal_next_frame": 493
125
+ },
126
+ {
127
+ "chunks": 53,
128
+ "demo": "demo_13",
129
+ "env_steps": 424,
130
+ "frames": 428,
131
+ "terminal_next_frame": 424
132
+ },
133
+ {
134
+ "chunks": 55,
135
+ "demo": "demo_14",
136
+ "env_steps": 435,
137
+ "frames": 439,
138
+ "terminal_next_frame": 435
139
+ },
140
+ {
141
+ "chunks": 59,
142
+ "demo": "demo_17",
143
+ "env_steps": 471,
144
+ "frames": 475,
145
+ "terminal_next_frame": 471
146
+ },
147
+ {
148
+ "chunks": 74,
149
+ "demo": "demo_20",
150
+ "env_steps": 590,
151
+ "frames": 594,
152
+ "terminal_next_frame": 590
153
+ },
154
+ {
155
+ "chunks": 63,
156
+ "demo": "demo_22",
157
+ "env_steps": 498,
158
+ "frames": 502,
159
+ "terminal_next_frame": 498
160
+ },
161
+ {
162
+ "chunks": 68,
163
+ "demo": "demo_28",
164
+ "env_steps": 544,
165
+ "frames": 548,
166
+ "terminal_next_frame": 544
167
+ },
168
+ {
169
+ "chunks": 66,
170
+ "demo": "demo_29",
171
+ "env_steps": 528,
172
+ "frames": 532,
173
+ "terminal_next_frame": 528
174
+ },
175
+ {
176
+ "chunks": 61,
177
+ "demo": "demo_33",
178
+ "env_steps": 486,
179
+ "frames": 490,
180
+ "terminal_next_frame": 486
181
+ },
182
+ {
183
+ "chunks": 60,
184
+ "demo": "demo_38",
185
+ "env_steps": 474,
186
+ "frames": 478,
187
+ "terminal_next_frame": 474
188
+ },
189
+ {
190
+ "chunks": 65,
191
+ "demo": "demo_39",
192
+ "env_steps": 513,
193
+ "frames": 517,
194
+ "terminal_next_frame": 513
195
+ },
196
+ {
197
+ "chunks": 61,
198
+ "demo": "demo_48",
199
+ "env_steps": 481,
200
+ "frames": 485,
201
+ "terminal_next_frame": 481
202
+ },
203
+ {
204
+ "chunks": 59,
205
+ "demo": "demo_49",
206
+ "env_steps": 468,
207
+ "frames": 474,
208
+ "terminal_next_frame": 468
209
+ },
210
+ {
211
+ "chunks": 69,
212
+ "demo": "demo_53",
213
+ "env_steps": 549,
214
+ "frames": 553,
215
+ "terminal_next_frame": 549
216
+ },
217
+ {
218
+ "chunks": 73,
219
+ "demo": "demo_59",
220
+ "env_steps": 582,
221
+ "frames": 586,
222
+ "terminal_next_frame": 582
223
+ },
224
+ {
225
+ "chunks": 84,
226
+ "demo": "demo_61",
227
+ "env_steps": 665,
228
+ "frames": 669,
229
+ "terminal_next_frame": 665
230
+ },
231
+ {
232
+ "chunks": 68,
233
+ "demo": "demo_69",
234
+ "env_steps": 544,
235
+ "frames": 548,
236
+ "terminal_next_frame": 544
237
+ },
238
+ {
239
+ "chunks": 56,
240
+ "demo": "demo_75",
241
+ "env_steps": 444,
242
+ "frames": 448,
243
+ "terminal_next_frame": 444
244
+ },
245
+ {
246
+ "chunks": 54,
247
+ "demo": "demo_76",
248
+ "env_steps": 430,
249
+ "frames": 434,
250
+ "terminal_next_frame": 430
251
+ },
252
+ {
253
+ "chunks": 55,
254
+ "demo": "demo_81",
255
+ "env_steps": 436,
256
+ "frames": 440,
257
+ "terminal_next_frame": 436
258
+ },
259
+ {
260
+ "chunks": 73,
261
+ "demo": "demo_83",
262
+ "env_steps": 579,
263
+ "frames": 583,
264
+ "terminal_next_frame": 579
265
+ },
266
+ {
267
+ "chunks": 59,
268
+ "demo": "demo_88",
269
+ "env_steps": 465,
270
+ "frames": 469,
271
+ "terminal_next_frame": 465
272
+ },
273
+ {
274
+ "chunks": 63,
275
+ "demo": "demo_95",
276
+ "env_steps": 500,
277
+ "frames": 504,
278
+ "terminal_next_frame": 500
279
+ },
280
+ {
281
+ "chunks": 61,
282
+ "demo": "demo_96",
283
+ "env_steps": 481,
284
+ "frames": 485,
285
+ "terminal_next_frame": 481
286
+ },
287
+ {
288
+ "chunks": 75,
289
+ "demo": "demo_102",
290
+ "env_steps": 595,
291
+ "frames": 599,
292
+ "terminal_next_frame": 595
293
+ },
294
+ {
295
+ "chunks": 61,
296
+ "demo": "demo_103",
297
+ "env_steps": 486,
298
+ "frames": 490,
299
+ "terminal_next_frame": 486
300
+ },
301
+ {
302
+ "chunks": 86,
303
+ "demo": "demo_105",
304
+ "env_steps": 681,
305
+ "frames": 688,
306
+ "terminal_next_frame": 681
307
+ },
308
+ {
309
+ "chunks": 61,
310
+ "demo": "demo_108",
311
+ "env_steps": 485,
312
+ "frames": 491,
313
+ "terminal_next_frame": 485
314
+ },
315
+ {
316
+ "chunks": 51,
317
+ "demo": "demo_119",
318
+ "env_steps": 404,
319
+ "frames": 408,
320
+ "terminal_next_frame": 404
321
+ },
322
+ {
323
+ "chunks": 52,
324
+ "demo": "demo_120",
325
+ "env_steps": 415,
326
+ "frames": 419,
327
+ "terminal_next_frame": 415
328
+ },
329
+ {
330
+ "chunks": 60,
331
+ "demo": "demo_126",
332
+ "env_steps": 480,
333
+ "frames": 484,
334
+ "terminal_next_frame": 480
335
+ },
336
+ {
337
+ "chunks": 65,
338
+ "demo": "demo_128",
339
+ "env_steps": 518,
340
+ "frames": 525,
341
+ "terminal_next_frame": 518
342
+ },
343
+ {
344
+ "chunks": 71,
345
+ "demo": "demo_139",
346
+ "env_steps": 565,
347
+ "frames": 569,
348
+ "terminal_next_frame": 565
349
+ },
350
+ {
351
+ "chunks": 53,
352
+ "demo": "demo_151",
353
+ "env_steps": 417,
354
+ "frames": 421,
355
+ "terminal_next_frame": 417
356
+ },
357
+ {
358
+ "chunks": 54,
359
+ "demo": "demo_152",
360
+ "env_steps": 425,
361
+ "frames": 429,
362
+ "terminal_next_frame": 425
363
+ },
364
+ {
365
+ "chunks": 68,
366
+ "demo": "demo_154",
367
+ "env_steps": 540,
368
+ "frames": 544,
369
+ "terminal_next_frame": 540
370
+ },
371
+ {
372
+ "chunks": 61,
373
+ "demo": "demo_159",
374
+ "env_steps": 486,
375
+ "frames": 490,
376
+ "terminal_next_frame": 486
377
+ },
378
+ {
379
+ "chunks": 64,
380
+ "demo": "demo_163",
381
+ "env_steps": 508,
382
+ "frames": 512,
383
+ "terminal_next_frame": 508
384
+ },
385
+ {
386
+ "chunks": 56,
387
+ "demo": "demo_165",
388
+ "env_steps": 442,
389
+ "frames": 446,
390
+ "terminal_next_frame": 442
391
+ },
392
+ {
393
+ "chunks": 57,
394
+ "demo": "demo_171",
395
+ "env_steps": 456,
396
+ "frames": 460,
397
+ "terminal_next_frame": 456
398
+ },
399
+ {
400
+ "chunks": 77,
401
+ "demo": "demo_173",
402
+ "env_steps": 615,
403
+ "frames": 619,
404
+ "terminal_next_frame": 615
405
+ },
406
+ {
407
+ "chunks": 52,
408
+ "demo": "demo_177",
409
+ "env_steps": 409,
410
+ "frames": 413,
411
+ "terminal_next_frame": 409
412
+ },
413
+ {
414
+ "chunks": 63,
415
+ "demo": "demo_180",
416
+ "env_steps": 502,
417
+ "frames": 506,
418
+ "terminal_next_frame": 502
419
+ },
420
+ {
421
+ "chunks": 70,
422
+ "demo": "demo_181",
423
+ "env_steps": 553,
424
+ "frames": 557,
425
+ "terminal_next_frame": 553
426
+ },
427
+ {
428
+ "chunks": 63,
429
+ "demo": "demo_182",
430
+ "env_steps": 497,
431
+ "frames": 501,
432
+ "terminal_next_frame": 497
433
+ },
434
+ {
435
+ "chunks": 58,
436
+ "demo": "demo_190",
437
+ "env_steps": 460,
438
+ "frames": 464,
439
+ "terminal_next_frame": 460
440
+ }
441
+ ],
442
+ "pixel_cache": "/ssd/WorkSpace/RL-Postrain/data/pixels/transport/v1",
443
+ "raw_coords": 353962,
444
+ "raw_coords_clipped": 0
445
+ },
446
+ "stats_hash": "4733037b2e1b5e648bb7974234a8c131f58cace23af3e078f5f6e997565346d6"
447
+ }
RL/Transport50/QC/diverse50/seed_2/resolved_config.json ADDED
@@ -0,0 +1,1127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[0930-S2DEC]",
14
+ "key": "learner.qc.agent.act_bounds_json",
15
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
16
+ "upstream": null,
17
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}"
18
+ },
19
+ {
20
+ "decision_ref": "[0930-S2DEC]",
21
+ "key": "learner.qc.agent.act_max",
22
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
23
+ "upstream": 1.0,
24
+ "value": 4.179931163787842
25
+ },
26
+ {
27
+ "decision_ref": "[0930-S2DEC]",
28
+ "key": "learner.qc.agent.act_min",
29
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
30
+ "upstream": -1.0,
31
+ "value": -3.978297472000122
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.actor_lr_partition",
36
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
37
+ "upstream": "lr=0.0003 shared by actor and critic",
38
+ "value": 0.0001
39
+ },
40
+ {
41
+ "decision_ref": "[1002-S2ALIGN]",
42
+ "key": "learner.qc.agent.actor_num_samples",
43
+ "reason": "single BoN knob including QC TD",
44
+ "upstream": 32,
45
+ "value": 16
46
+ },
47
+ {
48
+ "decision_ref": "[1002-S2ALIGN]",
49
+ "key": "learner.qc.agent.best_of_n",
50
+ "reason": "single BoN knob: collection/eval/QC TD",
51
+ "upstream": 8,
52
+ "value": 16
53
+ },
54
+ {
55
+ "decision_ref": "docs/scope.md [0928-I]",
56
+ "key": "learner.qc.agent.bon_agg",
57
+ "reason": "Best-of-N selection by the ensemble mean",
58
+ "upstream": null,
59
+ "value": "mean"
60
+ },
61
+ {
62
+ "decision_ref": "docs/scope.md [0928-I]",
63
+ "key": "learner.qc.agent.bon_sft",
64
+ "reason": "variant qc flag bon_sft",
65
+ "upstream": "upstream_yaml",
66
+ "value": false
67
+ },
68
+ {
69
+ "decision_ref": "[0930-S2DEC]",
70
+ "key": "learner.qc.agent.critic_lr_partition",
71
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
72
+ "upstream": "lr=0.0003 shared by actor and critic",
73
+ "value": 0.0003
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.partition_optimizer",
78
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
79
+ "upstream": null,
80
+ "value": true
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.q_target_agg",
85
+ "reason": "REDQ-style TD target: min over a random critic subset",
86
+ "upstream": null,
87
+ "value": "subset_min"
88
+ },
89
+ {
90
+ "decision_ref": "docs/scope.md [0928-I]",
91
+ "key": "learner.qc.agent.q_target_replace",
92
+ "reason": "subset drawn without replacement",
93
+ "upstream": null,
94
+ "value": false
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.q_target_subset",
99
+ "reason": "TD-target subset size",
100
+ "upstream": null,
101
+ "value": 2
102
+ },
103
+ {
104
+ "decision_ref": "[0930-S2DEC]",
105
+ "key": "learner.qc.agent.value_hidden_dims",
106
+ "reason": "unified.critic_hidden (critics only; actors native)",
107
+ "upstream": [
108
+ 512,
109
+ 512,
110
+ 512,
111
+ 512
112
+ ],
113
+ "value": [
114
+ 512,
115
+ 512,
116
+ 512
117
+ ]
118
+ },
119
+ {
120
+ "decision_ref": "docs/scope.md [0928-I]",
121
+ "key": "learner.qc.training.bc_pi_steps",
122
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
123
+ "upstream": 1000000,
124
+ "value": 0
125
+ },
126
+ {
127
+ "decision_ref": "docs/scope.md [0928-I]",
128
+ "key": "learner.qc.loop.buffer.utd_online",
129
+ "reason": "UTD=3 per executed action chunk",
130
+ "upstream": 1,
131
+ "value": "3 gradient updates per executed chunk"
132
+ },
133
+ {
134
+ "decision_ref": "docs/scope.md [0928-I]",
135
+ "key": "learner.qc.loop.n_step",
136
+ "reason": "n-step=3 counted in action chunks",
137
+ "upstream": "1 chunk (sequence_length = horizon_length)",
138
+ "value": "3 executed chunks (unified_env_step window)"
139
+ },
140
+ {
141
+ "decision_ref": "docs/scope.md [0928-I]",
142
+ "key": "learner.qc.loop.critic_actor_ratio",
143
+ "reason": "critic every update, actor every 2nd global update",
144
+ "upstream": "1:1 (utd_q = utd_pi)",
145
+ "value": "2:1"
146
+ },
147
+ {
148
+ "decision_ref": "docs/scope.md [0928-I]",
149
+ "key": "learner.qc.loop.buffer.start_training",
150
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
151
+ "upstream": 10000,
152
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
153
+ },
154
+ {
155
+ "decision_ref": "docs/scope.md [0928-I]",
156
+ "key": "learner.qc.loop.buffer.offline_ratio",
157
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
158
+ "upstream": 0.0,
159
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
160
+ },
161
+ {
162
+ "decision_ref": "docs/scope.md [0928-I]",
163
+ "key": "learner.qc.loop.buffer.size",
164
+ "reason": "one shared replay of executed chunks; success buffer = mask",
165
+ "upstream": 2000000,
166
+ "value": "1000000 executed chunks, shared FIFO"
167
+ },
168
+ {
169
+ "decision_ref": "docs/scope.md [0928-I]",
170
+ "key": "learner.qc.actor_architecture",
171
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
172
+ "upstream": [
173
+ 512,
174
+ 512,
175
+ 512,
176
+ 512
177
+ ],
178
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
179
+ },
180
+ {
181
+ "decision_ref": "docs/scope.md [0928-I]",
182
+ "key": "learner.qc.observation",
183
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
184
+ "upstream": "low-dim state",
185
+ "value": "frozen Ri features c [146] from the checkpoint encoder"
186
+ },
187
+ {
188
+ "decision_ref": "[0930-S2DEC]",
189
+ "key": "learner.qc.reward",
190
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
191
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
192
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
193
+ },
194
+ {
195
+ "decision_ref": "[0930-S2DEC]",
196
+ "key": "learner.qc.td_target",
197
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
198
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
199
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
200
+ },
201
+ {
202
+ "decision_ref": "[0930-S2DEC]",
203
+ "key": "learner.qc.eval",
204
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
205
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
206
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
207
+ },
208
+ {
209
+ "decision_ref": "[1002-S2ALIGN]",
210
+ "key": "learner.qc.lr_schedule",
211
+ "reason": "RL env-step clock shared with extra success TD",
212
+ "upstream": "native per-optimizer schedule",
213
+ "value": {
214
+ "actor": 0.0001,
215
+ "critic": 0.0003,
216
+ "decay_end_env_steps": 1606196,
217
+ "min_ratio": 0.1,
218
+ "schedule": "warmup_cosine",
219
+ "temperature": 0.0003,
220
+ "warmup_env_steps": 20000
221
+ }
222
+ },
223
+ {
224
+ "decision_ref": "[0930-S2DEC]",
225
+ "key": "learner.qc.agent.q_min_q_max",
226
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
227
+ "upstream": [
228
+ -999.9999999999991,
229
+ 0.0
230
+ ],
231
+ "value": [
232
+ 0.0,
233
+ 1.0
234
+ ]
235
+ }
236
+ ],
237
+ "protocol": {
238
+ "checkpoint_every_rounds": 5,
239
+ "checkpoint_retention": {
240
+ "best": true,
241
+ "collapse_drop": {
242
+ "min_abs": 0.2,
243
+ "ratio": 0.5
244
+ },
245
+ "diagnostics": true,
246
+ "every_evals": 3
247
+ },
248
+ "collection": {
249
+ "n_envs": 4,
250
+ "train_seed_base": 100000000,
251
+ "train_seed_stride": 1000000,
252
+ "training_freq_chunks": 1000
253
+ },
254
+ "convergence": {
255
+ "early_stop": false,
256
+ "gain_lag": 10,
257
+ "high_sr": 0.95,
258
+ "high_sr_evals": 5,
259
+ "ma_window": 10,
260
+ "max_env_steps": 1606196,
261
+ "min_env_steps": 200000,
262
+ "min_gain": 0.02,
263
+ "zero_success_evals": 8
264
+ },
265
+ "critic_ensemble": {
266
+ "num": 10,
267
+ "policy": "mean",
268
+ "target": {
269
+ "kind": "subset_min",
270
+ "replace": false,
271
+ "size": 2
272
+ }
273
+ },
274
+ "eval": {
275
+ "best_of_n": 16,
276
+ "episodes": 200,
277
+ "every_env_steps": 20000,
278
+ "include_step0": true,
279
+ "seed_start": 960000,
280
+ "workers": 4
281
+ },
282
+ "final_eval": {
283
+ "episodes": 300,
284
+ "on_zero_stop": true,
285
+ "seed_start": 800000
286
+ },
287
+ "n_step_chunks": 3,
288
+ "offline": {
289
+ "adaptive_ratio": false,
290
+ "cohort_size": 50,
291
+ "episodes": 50,
292
+ "learners_without_offline": [
293
+ "dsrl_sac"
294
+ ],
295
+ "ratio": {
296
+ "decay_env_steps": {
297
+ "can": 160000,
298
+ "square": 320000,
299
+ "tool_hang": 640000,
300
+ "transport": 320000
301
+ },
302
+ "end": 0.1,
303
+ "start": {
304
+ "can": 0.7,
305
+ "square": 0.5,
306
+ "tool_hang": 0.5,
307
+ "transport": 0.9
308
+ }
309
+ },
310
+ "selection": "ascending_demo_index",
311
+ "source": "bc_regime",
312
+ "unsupported_learners": [
313
+ "zprl"
314
+ ]
315
+ },
316
+ "replay": {
317
+ "capacity_chunks": 1000000,
318
+ "pin_start_successes": false,
319
+ "success_buffer": "success_mask_on_shared_fifo",
320
+ "success_skip_rule": "skip_below_success_batch_size",
321
+ "unit": "executed_chunk"
322
+ },
323
+ "start": {
324
+ "max_episodes": 200,
325
+ "mode": "offline_buffer",
326
+ "policy": {
327
+ "default": "base_cfm_ode",
328
+ "zprl": "vib_base"
329
+ },
330
+ "target_successes": 20
331
+ },
332
+ "unified": {
333
+ "actor_target_tau": 0.005,
334
+ "critic_hidden": [
335
+ 512,
336
+ 512,
337
+ 512
338
+ ],
339
+ "critic_target_every_critic_step": true,
340
+ "critic_target_tau": 0.005,
341
+ "dsrl_noise_imitation_reference_dims": 28,
342
+ "gamma_env": 0.999,
343
+ "lr": {
344
+ "actor": 0.0001,
345
+ "critic": 0.0003,
346
+ "decay_end_env_steps": 1606196,
347
+ "min_ratio": 0.1,
348
+ "schedule": "warmup_cosine",
349
+ "temperature": 0.0003,
350
+ "warmup_env_steps": 20000
351
+ },
352
+ "reward": "binary_first_success",
353
+ "success_td_polyak": true,
354
+ "timeout_bootstrap": false,
355
+ "zprl_imitation": "mse_pretanh_mean"
356
+ },
357
+ "updates": {
358
+ "batch_size": 256,
359
+ "critic_per_actor": 2,
360
+ "success_batch_size": 256,
361
+ "utd_per_chunk": 3
362
+ },
363
+ "version": "study2_protocol_v10"
364
+ },
365
+ "protocol_hash": "bdfdfb18fc67ba88af67e02b6592b190b509c5ad68cbd7357fcbca9fa8c796b9",
366
+ "provenance_extra": {
367
+ "agent_library": {
368
+ "framework": "jax",
369
+ "sources": {
370
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
371
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
372
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
373
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
374
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
375
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
376
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
377
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
378
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
379
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
380
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
381
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
382
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
383
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
384
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
385
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
386
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
387
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
388
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
389
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
390
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
391
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
392
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
393
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
394
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
395
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
396
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
397
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
398
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
399
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
400
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
401
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
402
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
403
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
404
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
405
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
406
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
407
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
408
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
409
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
410
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
411
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
412
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
413
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
414
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
415
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
416
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
417
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
418
+ "train.py": "d1ec607894c8e126e710b5a4ef6c48a4e9bf6c66c0872ce2e2bdc1c65b66d98d",
419
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
420
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
421
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
422
+ },
423
+ "version": "study2_jax_agents_v2"
424
+ },
425
+ "base_framework": "jax",
426
+ "bc_identity": {
427
+ "architecture": "dice_vision_resnet_flow_unet_v1",
428
+ "family": "StandardBaseBC",
429
+ "framework": "jax"
430
+ },
431
+ "checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261022_transport_focused50_v10_four/base/diverse50/epoch_3000.pkl",
432
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
433
+ "experiment": "transport_8",
434
+ "frozen_hash": "2414629d8dcbea1d75ca2831d5e0b649e65cffd56aa0ed22212fd0c3ec60b1bc",
435
+ "learner": {
436
+ "action_bounds": {
437
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
438
+ "compat_act_max": 4.179931163787842,
439
+ "compat_act_min": -3.978297472000122,
440
+ "hi": [
441
+ 1.0,
442
+ 1.0,
443
+ 1.0,
444
+ 1.0,
445
+ 2.775709390640259,
446
+ 2.2864010334014893,
447
+ 1.0,
448
+ 1.0,
449
+ 1.0,
450
+ 1.0,
451
+ 1.4753823280334473,
452
+ 4.088724613189697,
453
+ 4.179931163787842,
454
+ 1.0
455
+ ],
456
+ "kind": "per_dimension",
457
+ "lo": [
458
+ -1.0,
459
+ -1.0,
460
+ -1.0,
461
+ -1.4882197380065918,
462
+ -3.233746290206909,
463
+ -2.5405781269073486,
464
+ -1.0,
465
+ -1.0,
466
+ -1.0,
467
+ -1.0,
468
+ -1.7870254516601562,
469
+ -3.208747148513794,
470
+ -3.978297472000122,
471
+ -1.0
472
+ ],
473
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
474
+ },
475
+ "agent_config": {
476
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
477
+ "act_max": 4.179931163787842,
478
+ "act_min": -3.978297472000122,
479
+ "action_chunking": true,
480
+ "action_critic_hidden_dims": null,
481
+ "action_dim": 14,
482
+ "action_num_qs": 10,
483
+ "action_repeat": false,
484
+ "actor_backbone": "mlp",
485
+ "actor_decay_steps": 50000,
486
+ "actor_end_value": 2e-05,
487
+ "actor_hidden_dims": [
488
+ 512,
489
+ 512,
490
+ 512,
491
+ 512
492
+ ],
493
+ "actor_layer_norm": false,
494
+ "actor_lr": 0.0003,
495
+ "actor_lr_partition": 0.0001,
496
+ "actor_num_samples": 16,
497
+ "actor_obs": "state",
498
+ "actor_scheduler": "cosine",
499
+ "actor_tau": 0.005,
500
+ "actor_type": "best-of-n",
501
+ "actor_warmup_steps": 2000,
502
+ "actor_weight_decay": 0.0,
503
+ "adv_clip_min": null,
504
+ "agent_name": "acfql",
505
+ "alpha": 500,
506
+ "batch_size": 256,
507
+ "best_of_n": 16,
508
+ "bon_agg": "mean",
509
+ "bon_sft": false,
510
+ "bon_sft_coeff": 1.0,
511
+ "bootstrap_ratio": 0.5,
512
+ "bptt_num_samples": 4,
513
+ "calql_q_subsample": 2,
514
+ "clip_grad_norm": 1000.0,
515
+ "clip_intermediate_actions": true,
516
+ "clip_max_epsilon_multiplier": 1.0,
517
+ "clip_min_epsilon_multiplier": 1.0,
518
+ "constant_noise_std": 0.01,
519
+ "cql_alpha": 5.0,
520
+ "cql_n_actions": 10,
521
+ "cql_temp": 1.0,
522
+ "critic_backbone": "mlp",
523
+ "critic_decay_steps": 5000,
524
+ "critic_end_value": 1e-08,
525
+ "critic_loss_type": "mse",
526
+ "critic_lr": 0.0003,
527
+ "critic_lr_partition": 0.0003,
528
+ "critic_obs": "state",
529
+ "critic_scheduler": "constant",
530
+ "critic_warmup_steps": 500,
531
+ "critic_weight_decay": 1e-05,
532
+ "denoised_clip_value": 1.0,
533
+ "discount": 0.999,
534
+ "edit_action_scale": 0.5,
535
+ "edit_actor_hidden_dims": null,
536
+ "edit_hidden_dims": [
537
+ 256,
538
+ 256,
539
+ 256
540
+ ],
541
+ "edit_init_temperature": 1.0,
542
+ "edit_layer_norm": true,
543
+ "edit_lr": 0.0003,
544
+ "edit_policy_bound": 0.1,
545
+ "edit_start_step": 100000,
546
+ "edit_target_entropy": null,
547
+ "edit_weight_decay": 0.0,
548
+ "encoder": null,
549
+ "entropy_backup": false,
550
+ "flow_steps": 10,
551
+ "fql_distillation_coeff": 10.0,
552
+ "fql_train_main_policy": true,
553
+ "freeze_vision_encoder": false,
554
+ "ft_flow_steps": 10,
555
+ "horizon_length": 8,
556
+ "init_temperature": 1.0,
557
+ "layer_norm": true,
558
+ "lr": 0.0003,
559
+ "lr_action_critic": null,
560
+ "lr_actor": null,
561
+ "lr_actor_bc_flow": null,
562
+ "lr_critic": null,
563
+ "lr_edit_actor": null,
564
+ "lr_noise_critic": null,
565
+ "max_noise_std": 0.01,
566
+ "mc_regression": false,
567
+ "mc_regression_coeff": 1.0,
568
+ "min_noise_std": 0.01,
569
+ "mip_q_ensemble": false,
570
+ "mip_q_noise_scale": 1.0,
571
+ "mip_q_t_star": 0.9,
572
+ "mip_t_star": 0.999,
573
+ "muon_beta": 0.95,
574
+ "muon_nesterov": true,
575
+ "muon_ns_steps": 5,
576
+ "n_action_samples": 8,
577
+ "n_edit_samples": 8,
578
+ "noise_critic_grad_steps": 1,
579
+ "noise_dim": null,
580
+ "noise_net_hidden_dims": [
581
+ 256,
582
+ 256
583
+ ],
584
+ "noise_net_layer_norm": true,
585
+ "normalize_act_space_dimension": false,
586
+ "normalize_denoising_horizon": false,
587
+ "normalize_q_loss": false,
588
+ "num_bins": 256,
589
+ "num_ensemble_members": 10,
590
+ "num_qs": 10,
591
+ "ob_dims": [
592
+ 146
593
+ ],
594
+ "one_step_decay_steps": null,
595
+ "one_step_end_value": null,
596
+ "one_step_hidden_dims": [
597
+ 512,
598
+ 512,
599
+ 512,
600
+ 512
601
+ ],
602
+ "one_step_layer_norm": false,
603
+ "one_step_lr": null,
604
+ "one_step_scheduler": null,
605
+ "one_step_warmup_steps": null,
606
+ "optimizer_action_critic": null,
607
+ "optimizer_actor": "adam",
608
+ "optimizer_actor_bc_flow": "adam",
609
+ "optimizer_critic": "adam",
610
+ "optimizer_edit_actor": null,
611
+ "paligemma_tokenizer_path": null,
612
+ "partition_optimizer": true,
613
+ "policy_type": "flow",
614
+ "ppo_batch_size": 256,
615
+ "ppo_lr": 4.5e-05,
616
+ "proprio_dim": null,
617
+ "q_agg": "mean",
618
+ "q_max": 1.0,
619
+ "q_min": 0.0,
620
+ "q_target_agg": "subset_min",
621
+ "q_target_replace": false,
622
+ "q_target_subset": 2,
623
+ "q_variance_reduction": false,
624
+ "q_vr_num_samples": 8,
625
+ "q_vr_reduction": "mean",
626
+ "randn_clip_value": 3.0,
627
+ "shortcut_denoising_steps": 4,
628
+ "shortcut_inference_steps": 4,
629
+ "simba_actor_hidden_dims": [
630
+ 1024,
631
+ 1024
632
+ ],
633
+ "simba_critic_hidden_dims": [
634
+ 1024,
635
+ 1024,
636
+ 1024
637
+ ],
638
+ "subsample_bon": false,
639
+ "target_entropy": null,
640
+ "tau": 0.005,
641
+ "tf_pi_conditioning": "adaln",
642
+ "tf_pi_dropout": 0.0,
643
+ "tf_pi_embed_dim": 256,
644
+ "tf_pi_heads": 4,
645
+ "tf_pi_layers": 8,
646
+ "tf_q_conditioning": "adaln",
647
+ "tf_q_dropout": 0.0,
648
+ "tf_q_embed_dim": 256,
649
+ "tf_q_heads": 4,
650
+ "tf_q_layers": 8,
651
+ "time_embedding": "sinusoidal",
652
+ "time_embedding_dim": 32,
653
+ "train_one_step_in_offline": true,
654
+ "train_one_step_in_online": true,
655
+ "useFiLM": false,
656
+ "useSimBa": false,
657
+ "useTF": false,
658
+ "use_constant_noise": true,
659
+ "use_constant_scheduler_for_bc": false,
660
+ "use_denoiser": false,
661
+ "use_edit_actor": false,
662
+ "use_mip_q": false,
663
+ "use_muon": false,
664
+ "use_one_step_policy": false,
665
+ "use_shortcut": false,
666
+ "use_state": "proprio",
667
+ "use_success_buffer_q": false,
668
+ "value_hidden_dims": [
669
+ 512,
670
+ 512,
671
+ 512
672
+ ],
673
+ "vit_encoder": "minvit",
674
+ "vit_img_proj_dim": 0,
675
+ "vit_state_proj_dim": 0
676
+ },
677
+ "convention": "unified_env_step",
678
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
679
+ "gamma_env": 0.999,
680
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
681
+ "learner": "qc",
682
+ "optimizers": {
683
+ "actor_lr": 0.0001,
684
+ "critic_lr": 0.0003,
685
+ "reset_optimizers_with_lr": "not used",
686
+ "schedule": {
687
+ "actor": 0.0001,
688
+ "critic": 0.0003,
689
+ "decay_end_env_steps": 1606196,
690
+ "min_ratio": 0.1,
691
+ "schedule": "warmup_cosine",
692
+ "temperature": 0.0003,
693
+ "warmup_env_steps": 20000
694
+ }
695
+ },
696
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
697
+ "recipe": {
698
+ "derived": {
699
+ "agent.actor_num_samples": {
700
+ "after": 8,
701
+ "before": 32
702
+ },
703
+ "agent.actor_type": {
704
+ "after": "best-of-n",
705
+ "before": "upstream_yaml"
706
+ },
707
+ "agent.bon_sft": {
708
+ "after": false,
709
+ "before": "upstream_yaml"
710
+ }
711
+ },
712
+ "explicit_overrides": {},
713
+ "source": "third_party/ogpo/scripts/qc/transport.sh",
714
+ "upstream_exact": false
715
+ },
716
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
717
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
718
+ "scan_block": 100,
719
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
720
+ "success_terms": {
721
+ "bc_on_success_batch": false,
722
+ "critic_success_td": false,
723
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
724
+ },
725
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=3 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
726
+ "unified_notes": {
727
+ "critic_target_every_critic_step": {
728
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
729
+ "value": true
730
+ },
731
+ "success_td_polyak": {
732
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
733
+ "value": true
734
+ }
735
+ },
736
+ "upstream_config": {
737
+ "agent": {
738
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -1.4882197380065918, -3.233746290206909, -2.5405781269073486, -1.0, -1.0, -1.0, -1.0, -1.7870254516601562, -3.208747148513794, -3.978297472000122, -1.0], \"hi\": [1.0, 1.0, 1.0, 1.0, 2.775709390640259, 2.2864010334014893, 1.0, 1.0, 1.0, 1.0, 1.4753823280334473, 4.088724613189697, 4.179931163787842, 1.0]}",
739
+ "act_max": 4.179931163787842,
740
+ "act_min": -3.978297472000122,
741
+ "action_chunking": true,
742
+ "action_critic_hidden_dims": null,
743
+ "action_dim": null,
744
+ "action_num_qs": 10,
745
+ "action_repeat": false,
746
+ "actor_backbone": "mlp",
747
+ "actor_decay_steps": 50000,
748
+ "actor_end_value": 2e-05,
749
+ "actor_hidden_dims": [
750
+ 512,
751
+ 512,
752
+ 512,
753
+ 512
754
+ ],
755
+ "actor_layer_norm": false,
756
+ "actor_lr": 0.0003,
757
+ "actor_lr_partition": 0.0001,
758
+ "actor_num_samples": 16,
759
+ "actor_obs": "state",
760
+ "actor_scheduler": "cosine",
761
+ "actor_tau": 0.005,
762
+ "actor_type": "best-of-n",
763
+ "actor_warmup_steps": 2000,
764
+ "actor_weight_decay": 0.0,
765
+ "adv_clip_min": null,
766
+ "agent_name": "acfql",
767
+ "alpha": 500,
768
+ "batch_size": 256,
769
+ "best_of_n": 16,
770
+ "bon_agg": "mean",
771
+ "bon_sft": false,
772
+ "bon_sft_coeff": 1.0,
773
+ "bootstrap_ratio": 0.5,
774
+ "bptt_num_samples": 4,
775
+ "calql_q_subsample": 2,
776
+ "clip_grad_norm": 1000.0,
777
+ "clip_intermediate_actions": true,
778
+ "clip_max_epsilon_multiplier": 1.0,
779
+ "clip_min_epsilon_multiplier": 1.0,
780
+ "constant_noise_std": 0.01,
781
+ "cql_alpha": 5.0,
782
+ "cql_n_actions": 10,
783
+ "cql_temp": 1.0,
784
+ "critic_backbone": "mlp",
785
+ "critic_decay_steps": 5000,
786
+ "critic_end_value": 1e-08,
787
+ "critic_loss_type": "mse",
788
+ "critic_lr": 0.0003,
789
+ "critic_lr_partition": 0.0003,
790
+ "critic_obs": "state",
791
+ "critic_scheduler": "constant",
792
+ "critic_warmup_steps": 500,
793
+ "critic_weight_decay": 1e-05,
794
+ "denoised_clip_value": 1.0,
795
+ "discount": 0.999,
796
+ "edit_action_scale": 0.5,
797
+ "edit_actor_hidden_dims": null,
798
+ "edit_hidden_dims": [
799
+ 256,
800
+ 256,
801
+ 256
802
+ ],
803
+ "edit_init_temperature": 1.0,
804
+ "edit_layer_norm": true,
805
+ "edit_lr": 0.0003,
806
+ "edit_policy_bound": 0.1,
807
+ "edit_start_step": 100000,
808
+ "edit_target_entropy": null,
809
+ "edit_weight_decay": 0.0,
810
+ "encoder": null,
811
+ "entropy_backup": false,
812
+ "flow_steps": 10,
813
+ "fql_distillation_coeff": 10.0,
814
+ "fql_train_main_policy": true,
815
+ "freeze_vision_encoder": false,
816
+ "ft_flow_steps": 10,
817
+ "horizon_length": 8,
818
+ "init_temperature": 1.0,
819
+ "layer_norm": true,
820
+ "lr": 0.0003,
821
+ "lr_action_critic": null,
822
+ "lr_actor": null,
823
+ "lr_actor_bc_flow": null,
824
+ "lr_critic": null,
825
+ "lr_edit_actor": null,
826
+ "lr_noise_critic": null,
827
+ "max_noise_std": 0.01,
828
+ "mc_regression": false,
829
+ "mc_regression_coeff": 1.0,
830
+ "min_noise_std": 0.01,
831
+ "mip_q_ensemble": false,
832
+ "mip_q_noise_scale": 1.0,
833
+ "mip_q_t_star": 0.9,
834
+ "mip_t_star": 0.999,
835
+ "muon_beta": 0.95,
836
+ "muon_nesterov": true,
837
+ "muon_ns_steps": 5,
838
+ "n_action_samples": 8,
839
+ "n_edit_samples": 8,
840
+ "noise_critic_grad_steps": 1,
841
+ "noise_dim": null,
842
+ "noise_net_hidden_dims": [
843
+ 256,
844
+ 256
845
+ ],
846
+ "noise_net_layer_norm": true,
847
+ "normalize_act_space_dimension": false,
848
+ "normalize_denoising_horizon": false,
849
+ "normalize_q_loss": false,
850
+ "num_bins": 256,
851
+ "num_ensemble_members": 10,
852
+ "num_qs": 10,
853
+ "ob_dims": null,
854
+ "one_step_decay_steps": null,
855
+ "one_step_end_value": null,
856
+ "one_step_hidden_dims": [
857
+ 512,
858
+ 512,
859
+ 512,
860
+ 512
861
+ ],
862
+ "one_step_layer_norm": false,
863
+ "one_step_lr": null,
864
+ "one_step_scheduler": null,
865
+ "one_step_warmup_steps": null,
866
+ "optimizer_action_critic": null,
867
+ "optimizer_actor": "adam",
868
+ "optimizer_actor_bc_flow": "adam",
869
+ "optimizer_critic": "adam",
870
+ "optimizer_edit_actor": null,
871
+ "paligemma_tokenizer_path": null,
872
+ "partition_optimizer": true,
873
+ "policy_type": "flow",
874
+ "ppo_batch_size": 256,
875
+ "ppo_lr": 4.5e-05,
876
+ "proprio_dim": null,
877
+ "q_agg": "mean",
878
+ "q_max": null,
879
+ "q_min": null,
880
+ "q_target_agg": "subset_min",
881
+ "q_target_replace": false,
882
+ "q_target_subset": 2,
883
+ "q_variance_reduction": false,
884
+ "q_vr_num_samples": 8,
885
+ "q_vr_reduction": "mean",
886
+ "randn_clip_value": 3.0,
887
+ "shortcut_denoising_steps": 4,
888
+ "shortcut_inference_steps": 4,
889
+ "simba_actor_hidden_dims": [
890
+ 1024,
891
+ 1024
892
+ ],
893
+ "simba_critic_hidden_dims": [
894
+ 1024,
895
+ 1024,
896
+ 1024
897
+ ],
898
+ "subsample_bon": false,
899
+ "target_entropy": null,
900
+ "tau": 0.005,
901
+ "tf_pi_conditioning": "adaln",
902
+ "tf_pi_dropout": 0.0,
903
+ "tf_pi_embed_dim": 256,
904
+ "tf_pi_heads": 4,
905
+ "tf_pi_layers": 8,
906
+ "tf_q_conditioning": "adaln",
907
+ "tf_q_dropout": 0.0,
908
+ "tf_q_embed_dim": 256,
909
+ "tf_q_heads": 4,
910
+ "tf_q_layers": 8,
911
+ "time_embedding": "sinusoidal",
912
+ "time_embedding_dim": 32,
913
+ "train_one_step_in_offline": true,
914
+ "train_one_step_in_online": true,
915
+ "useFiLM": false,
916
+ "useSimBa": false,
917
+ "useTF": false,
918
+ "use_constant_noise": true,
919
+ "use_constant_scheduler_for_bc": false,
920
+ "use_denoiser": false,
921
+ "use_edit_actor": false,
922
+ "use_mip_q": false,
923
+ "use_muon": false,
924
+ "use_one_step_policy": false,
925
+ "use_shortcut": false,
926
+ "use_state": "proprio",
927
+ "use_success_buffer_q": false,
928
+ "value_hidden_dims": [
929
+ 512,
930
+ 512,
931
+ 512
932
+ ],
933
+ "vit_encoder": "minvit",
934
+ "vit_img_proj_dim": 0,
935
+ "vit_state_proj_dim": 0
936
+ },
937
+ "bc": {
938
+ "clip_bc": false,
939
+ "clip_bc_threshold": 0.45,
940
+ "clip_bc_wrt": "ODE"
941
+ },
942
+ "buffer": {
943
+ "offline_ratio": 0.0,
944
+ "size": 2000000,
945
+ "start_training": 10000,
946
+ "utd_online": 1,
947
+ "utd_pi": null,
948
+ "utd_q": null,
949
+ "utd_v": null,
950
+ "utd_warmup": 1
951
+ },
952
+ "checkpoint": {
953
+ "ep_resume": 0,
954
+ "restore_actor_path": null,
955
+ "restore_critic_path": null,
956
+ "restore_encoder_path": null
957
+ },
958
+ "dataset": {
959
+ "num_offline_trajs": -1,
960
+ "ogbench_dir": null,
961
+ "p_aug": 0.0,
962
+ "proportion": 1.0,
963
+ "replace_interval": 1000
964
+ },
965
+ "env": {
966
+ "name": "transport-mh-low_dim",
967
+ "post_success_steps": 0,
968
+ "rew_fn": "sparse",
969
+ "sparse": false
970
+ },
971
+ "eval": {
972
+ "eval_episodes": 2,
973
+ "n_eval_envs": 32,
974
+ "plot_q_vs_mc": false,
975
+ "video_episodes": 0,
976
+ "video_frame_skip": 3
977
+ },
978
+ "experiment": {
979
+ "log": true,
980
+ "project": "QC",
981
+ "run_group": "transport",
982
+ "run_name": "qc_transport",
983
+ "save_dir": "exp/ogpo",
984
+ "seed": 1,
985
+ "wandb_name": "qc_transport"
986
+ },
987
+ "intervals": {
988
+ "eval_interval": 20000,
989
+ "eval_interval_bc": 5000,
990
+ "log_interval": 5000,
991
+ "preemption_backup_interval": -1,
992
+ "save_interval": 500000,
993
+ "save_interval_bc": -1,
994
+ "step_data_log_interval": -1
995
+ },
996
+ "success_buffer": {
997
+ "batch_size": 256,
998
+ "use_success_buffer": false
999
+ },
1000
+ "training": {
1001
+ "bc_only": false,
1002
+ "bc_pi_steps": 0,
1003
+ "bc_q_steps": 0,
1004
+ "bc_refine_steps": 0,
1005
+ "calql_steps": 0,
1006
+ "force_jax_sync": true,
1007
+ "gpu_mode": "single",
1008
+ "online_steps": 6000000,
1009
+ "q_warmup_steps": 0
1010
+ }
1011
+ },
1012
+ "upstream_source": {
1013
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1014
+ "patches_sha256": {
1015
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1016
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1017
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1018
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1019
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1020
+ }
1021
+ }
1022
+ },
1023
+ "observation_mode": "pixel_state",
1024
+ "offline_buffer": {
1025
+ "actions_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1026
+ "aux_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
1027
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
1028
+ "chunks": 3183,
1029
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1030
+ "decision_ref": "[1017-S2OFFBC]",
1031
+ "env_steps": 25283,
1032
+ "features_sha256": "9f82519c4cb371ffbbb2c3f5ae9208cb7dc6e48ca9e0ba28b7c3845dc1f1148f",
1033
+ "horizon": 8,
1034
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
1035
+ "learner_coordinates": "JaxStudy2Learner:qc",
1036
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1037
+ "ratio": {
1038
+ "adaptive_ratio": false,
1039
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1040
+ },
1041
+ "selection": {
1042
+ "cohort_size": 50,
1043
+ "demos": [
1044
+ "demo_0",
1045
+ "demo_7",
1046
+ "demo_8",
1047
+ "demo_10",
1048
+ "demo_11",
1049
+ "demo_13",
1050
+ "demo_14",
1051
+ "demo_17",
1052
+ "demo_20",
1053
+ "demo_22",
1054
+ "demo_28",
1055
+ "demo_29",
1056
+ "demo_33",
1057
+ "demo_38",
1058
+ "demo_39",
1059
+ "demo_48",
1060
+ "demo_49",
1061
+ "demo_53",
1062
+ "demo_59",
1063
+ "demo_61",
1064
+ "demo_69",
1065
+ "demo_75",
1066
+ "demo_76",
1067
+ "demo_81",
1068
+ "demo_83",
1069
+ "demo_88",
1070
+ "demo_95",
1071
+ "demo_96",
1072
+ "demo_102",
1073
+ "demo_103",
1074
+ "demo_105",
1075
+ "demo_108",
1076
+ "demo_119",
1077
+ "demo_120",
1078
+ "demo_126",
1079
+ "demo_128",
1080
+ "demo_139",
1081
+ "demo_151",
1082
+ "demo_152",
1083
+ "demo_154",
1084
+ "demo_159",
1085
+ "demo_163",
1086
+ "demo_165",
1087
+ "demo_171",
1088
+ "demo_173",
1089
+ "demo_177",
1090
+ "demo_180",
1091
+ "demo_181",
1092
+ "demo_182",
1093
+ "demo_190"
1094
+ ],
1095
+ "manifest": "/ssd/WorkSpace/RL-Postrain/data/bc/versions/ph_distribution_v2/manifests/transport.json",
1096
+ "manifest_sha256": "8f2569c5918df85bccd661f8b4c8f76fd73948ca0c4ff96487e949ddbdb4013f",
1097
+ "regime": "diverse50",
1098
+ "source": "bc_regime",
1099
+ "source_ph_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
1100
+ "split": {
1101
+ "training": 45,
1102
+ "validation": 5
1103
+ },
1104
+ "task": "transport"
1105
+ }
1106
+ },
1107
+ "offline_role": "mixed",
1108
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
1109
+ "smoke": false,
1110
+ "task": "transport",
1111
+ "task_spec_hash": "fd9c198a613abab85d95bda9521355ae6d926bcf7da1895ca7a1dd03325117cb"
1112
+ },
1113
+ "seed": 2,
1114
+ "start_policy": "base_cfm_ode",
1115
+ "variant": {
1116
+ "flags": {
1117
+ "bon_sft": false,
1118
+ "use_success_buffer": false
1119
+ },
1120
+ "learner": "qc",
1121
+ "name": "qc",
1122
+ "phase": 1,
1123
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1124
+ "version": "study2_variants_v4"
1125
+ },
1126
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1127
+ }
RL/Transport50/QC/diverse50/seed_2/start_protocol.json ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "committed_episodes": [],
3
+ "mode": "offline_buffer",
4
+ "note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
5
+ "offline": {
6
+ "adaptive_ratio": false,
7
+ "batches": 0,
8
+ "decay_env_steps": null,
9
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
10
+ "last_ratio": null,
11
+ "offline_rows": 0,
12
+ "online_rows": 0,
13
+ "ratio_end": null,
14
+ "ratio_start": null,
15
+ "records": 3183
16
+ },
17
+ "offline_identity": {
18
+ "actions_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
19
+ "aux_sha256": "602a4cc6482db479799da53e62fa1678116093a2158758a060c4890768d8a642",
20
+ "checkpoint_sha256": "b82b0c433a011cd236f7071cf7d169f297f77ee51ba490f305b0b25754cd747c",
21
+ "chunks": 3183,
22
+ "dataset_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
23
+ "decision_ref": "[1017-S2OFFBC]",
24
+ "env_steps": 25283,
25
+ "features_sha256": "9f82519c4cb371ffbbb2c3f5ae9208cb7dc6e48ca9e0ba28b7c3845dc1f1148f",
26
+ "horizon": 8,
27
+ "learner_coordinates": "JaxStudy2Learner:qc",
28
+ "pixel_spec_hash": "5aa40c11f734d167f60d1c52f395d200b6a42fa0ea156bcc61d0ed317cd8335b",
29
+ "ratio": {
30
+ "adaptive_ratio": false,
31
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
32
+ },
33
+ "selection": {
34
+ "cohort_size": 50,
35
+ "demos": [
36
+ "demo_0",
37
+ "demo_7",
38
+ "demo_8",
39
+ "demo_10",
40
+ "demo_11",
41
+ "demo_13",
42
+ "demo_14",
43
+ "demo_17",
44
+ "demo_20",
45
+ "demo_22",
46
+ "demo_28",
47
+ "demo_29",
48
+ "demo_33",
49
+ "demo_38",
50
+ "demo_39",
51
+ "demo_48",
52
+ "demo_49",
53
+ "demo_53",
54
+ "demo_59",
55
+ "demo_61",
56
+ "demo_69",
57
+ "demo_75",
58
+ "demo_76",
59
+ "demo_81",
60
+ "demo_83",
61
+ "demo_88",
62
+ "demo_95",
63
+ "demo_96",
64
+ "demo_102",
65
+ "demo_103",
66
+ "demo_105",
67
+ "demo_108",
68
+ "demo_119",
69
+ "demo_120",
70
+ "demo_126",
71
+ "demo_128",
72
+ "demo_139",
73
+ "demo_151",
74
+ "demo_152",
75
+ "demo_154",
76
+ "demo_159",
77
+ "demo_163",
78
+ "demo_165",
79
+ "demo_171",
80
+ "demo_173",
81
+ "demo_177",
82
+ "demo_180",
83
+ "demo_181",
84
+ "demo_182",
85
+ "demo_190"
86
+ ],
87
+ "manifest": "/ssd/WorkSpace/RL-Postrain/data/bc/versions/ph_distribution_v2/manifests/transport.json",
88
+ "manifest_sha256": "8f2569c5918df85bccd661f8b4c8f76fd73948ca0c4ff96487e949ddbdb4013f",
89
+ "regime": "diverse50",
90
+ "source": "bc_regime",
91
+ "source_ph_sha256": "260515618f4c8b660e54171497ecceb08c7c6463691a9ce862b158bcacd950b4",
92
+ "split": {
93
+ "training": 45,
94
+ "validation": 5
95
+ },
96
+ "task": "transport"
97
+ }
98
+ },
99
+ "policy": "base_cfm_ode",
100
+ "start_chunks": 0,
101
+ "start_env_steps": 0,
102
+ "status": "complete"
103
+ }
RL/Transport50/QC/diverse50/seed_2/status.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accounting": {
3
+ "evaluation": {
4
+ "calls": 82,
5
+ "episodes": 16500
6
+ },
7
+ "history_complete": true,
8
+ "normalized": {
9
+ "checkpoint_seconds_per_env_step": 4.759588210013975e-05,
10
+ "chunks_per_10000_env_steps": 1254.2369303234132,
11
+ "denominator_env_steps": 1610541,
12
+ "diagnostics_seconds_per_env_step": 0.00010717379444627371,
13
+ "episodes_per_10000_env_steps": 16.007043595909696,
14
+ "evaluation_seconds_per_env_step": 0.07578265550351154,
15
+ "history_complete": true,
16
+ "projected_chunks": 201455.03405377446,
17
+ "projected_episodes": 2571.0449395575774,
18
+ "projected_training_seconds": 38745.08536003422,
19
+ "projected_wall_seconds": 160780.75128762456,
20
+ "projection_budget_env_steps": 1606196,
21
+ "projection_is_estimate": true,
22
+ "rollout_seconds_per_env_step": 0.011849154830411822,
23
+ "training_seconds_per_env_step": 0.02412226487927639,
24
+ "updates_seconds_per_env_step": 0.012273110048864569,
25
+ "wall_seconds_per_env_step": 0.10010033102287925
26
+ },
27
+ "rollout": {
28
+ "chunks": 202000,
29
+ "env_steps": 1610541,
30
+ "episodes": 2578,
31
+ "start_chunks": 0,
32
+ "start_env_steps": 0,
33
+ "successes": 1321,
34
+ "truncated_episodes": 1257
35
+ },
36
+ "schema_version": 1,
37
+ "sequence": 1733,
38
+ "sessions": 1,
39
+ "time": {
40
+ "checkpoint_seconds": 76.65511955344118,
41
+ "diagnostics_seconds": 172.60779008129612,
42
+ "evaluation_seconds": 122051.07377728098,
43
+ "preparation_seconds": 60.451391709037125,
44
+ "restore_seconds": 0.0,
45
+ "rollout_seconds": 19083.549669726286,
46
+ "start_rollout_seconds": 0.0,
47
+ "training_seconds": 38849.89660093468,
48
+ "updates_seconds": 19766.34693120839,
49
+ "wall_hours": 44.78213534053305,
50
+ "wall_seconds": 161215.68722591898
51
+ }
52
+ },
53
+ "detail": null,
54
+ "evals": 81,
55
+ "final_eval": {
56
+ "episodes": 300,
57
+ "imputed": false,
58
+ "n_success": 156,
59
+ "observed": true,
60
+ "seed_start": 800000,
61
+ "sr": 0.52
62
+ },
63
+ "learner": "qc",
64
+ "learner_counters": {
65
+ "actor_updates": 303000,
66
+ "actor_updates_applied": 303000,
67
+ "actor_updates_skipped": 0,
68
+ "clip_stats": {
69
+ "clip_mag_sum": 4422.615698337555,
70
+ "coords_clipped_frac": 0.02849818776520509,
71
+ "max_clip_mag": 0.529962420463562,
72
+ "mean_clip_mag": 0.006859501690344145,
73
+ "n_clipped": 644743,
74
+ "n_coords": 22624000
75
+ },
76
+ "clip_stats_start": {
77
+ "clip_mag_sum": 0.0,
78
+ "coords_clipped_frac": 0.0,
79
+ "max_clip_mag": 0.0,
80
+ "mean_clip_mag": 0.0,
81
+ "n_clipped": 0,
82
+ "n_coords": 0
83
+ },
84
+ "critic_updates": 606000,
85
+ "policy_version": 202,
86
+ "success_missing": 0
87
+ },
88
+ "phase": 1,
89
+ "protocol_hash": "bdfdfb18fc67ba88af67e02b6592b190b509c5ad68cbd7357fcbca9fa8c796b9",
90
+ "resumes": [],
91
+ "schedule": {
92
+ "actor_updates": 303000,
93
+ "chunk_transitions": 202000,
94
+ "critic_updates": 606000,
95
+ "gradient_updates": 606000,
96
+ "rl_chunks": 202000,
97
+ "rl_env_steps": 1610541,
98
+ "round_index": 202,
99
+ "update_index": 606000
100
+ },
101
+ "seed": 2,
102
+ "start": {
103
+ "mode": "offline_buffer",
104
+ "note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
105
+ "offline": {
106
+ "adaptive_ratio": false,
107
+ "batches": 606000,
108
+ "decay_env_steps": null,
109
+ "identity_hash": "1b952ebfc6cb32270d175b598a4eb3bb17e75cc6c7ad9f0a28405f66f1b13c61",
110
+ "last_ratio": 0.015513888415029415,
111
+ "offline_rows": 9836080,
112
+ "online_rows": 145299920,
113
+ "ratio_end": null,
114
+ "ratio_start": null,
115
+ "records": 3183
116
+ },
117
+ "start_chunks": 0,
118
+ "start_env_steps": 0,
119
+ "status": "complete"
120
+ },
121
+ "status": "budget_complete",
122
+ "stop": {
123
+ "eval_index": 80,
124
+ "gain": 0.005499999999999838,
125
+ "ma": 0.5479999999999999,
126
+ "rl_env_steps": 1610541,
127
+ "sr": 0.54,
128
+ "status": "budget_complete",
129
+ "trend": "flat"
130
+ },
131
+ "success_sampling": {
132
+ "imitation": {
133
+ "requested": 0,
134
+ "rounds_with_skips": 0,
135
+ "sampled": 0,
136
+ "skipped": 0
137
+ },
138
+ "last_round": {
139
+ "imitation": {
140
+ "requested": 0,
141
+ "sampled": 0,
142
+ "skipped": 0
143
+ },
144
+ "requested": 0,
145
+ "round": 201,
146
+ "sampled": 0,
147
+ "skipped": 0
148
+ },
149
+ "requested": 0,
150
+ "rounds_with_skips": 0,
151
+ "sampled": 0,
152
+ "skipped": 0,
153
+ "td": {
154
+ "requested": 0,
155
+ "rounds_with_skips": 0,
156
+ "sampled": 0,
157
+ "skipped": 0
158
+ }
159
+ },
160
+ "variant": "qc",
161
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
162
+ "wandb": {
163
+ "enabled": false,
164
+ "error": null,
165
+ "finished": true,
166
+ "mode": "online",
167
+ "name": "Transport-Diverse50-QC",
168
+ "project": "RL-postTrain",
169
+ "reason": null,
170
+ "run_id": "15e00de4",
171
+ "transport": "relay",
172
+ "url": "https://wandb.ai/f70331658-university-of-chinese-academy-of-sciences/RL-postTrain/runs/15e00de4"
173
+ }
174
+ }
RL/Transport50/QC/diverse50/seed_2/training_accounting.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "evaluation": {
3
+ "calls": 82,
4
+ "episodes": 16500
5
+ },
6
+ "history_complete": true,
7
+ "normalized": {
8
+ "checkpoint_seconds_per_env_step": 4.759588210013975e-05,
9
+ "chunks_per_10000_env_steps": 1254.2369303234132,
10
+ "denominator_env_steps": 1610541,
11
+ "diagnostics_seconds_per_env_step": 0.00010717379444627371,
12
+ "episodes_per_10000_env_steps": 16.007043595909696,
13
+ "evaluation_seconds_per_env_step": 0.07578265550351154,
14
+ "history_complete": true,
15
+ "projected_chunks": 201455.03405377446,
16
+ "projected_episodes": 2571.0449395575774,
17
+ "projected_training_seconds": 38745.08536003422,
18
+ "projected_wall_seconds": 160780.75128762456,
19
+ "projection_budget_env_steps": 1606196,
20
+ "projection_is_estimate": true,
21
+ "rollout_seconds_per_env_step": 0.011849154830411822,
22
+ "training_seconds_per_env_step": 0.02412226487927639,
23
+ "updates_seconds_per_env_step": 0.012273110048864569,
24
+ "wall_seconds_per_env_step": 0.10010033102287925
25
+ },
26
+ "rollout": {
27
+ "chunks": 202000,
28
+ "env_steps": 1610541,
29
+ "episodes": 2578,
30
+ "start_chunks": 0,
31
+ "start_env_steps": 0,
32
+ "successes": 1321,
33
+ "truncated_episodes": 1257
34
+ },
35
+ "schema_version": 1,
36
+ "sequence": 1734,
37
+ "sessions": 1,
38
+ "time": {
39
+ "checkpoint_seconds": 76.65511955344118,
40
+ "diagnostics_seconds": 172.60779008129612,
41
+ "evaluation_seconds": 122051.07377728098,
42
+ "preparation_seconds": 60.451391709037125,
43
+ "restore_seconds": 0.0,
44
+ "rollout_seconds": 19083.549669726286,
45
+ "start_rollout_seconds": 0.0,
46
+ "training_seconds": 38849.89660093468,
47
+ "updates_seconds": 19766.34693120839,
48
+ "wall_hours": 44.78213534053305,
49
+ "wall_seconds": 161215.68722591898
50
+ }
51
+ }