yangfy0627 commited on
Commit
4c8a522
·
verified ·
1 Parent(s): 95320f4

Publish completed ToolHang100 Mixed100 QC v10 seed1 final and best checkpoints

Browse files
RL/ToolHang100/QC/mixed100/seed_1/SHA256SUMS ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 4477959f523302128df56a191755dc774523605c58dd005d12b55c03f9933f6d best/source_metadata.json
2
+ bc96fb09cf2841fda4b9b5e2eca95950e4fe4a6bbcfe22862975d2a25638a494 best/source_resolved_config.json
3
+ 6ddafc0ff69f6fdea7c2951cad4b6ba09f3e32375fd411f9cd7abbec81adb8d9 best/state.pkl
4
+ 94e1ac5137abcfda05e57298874937bba34e6d36081e25b6c69b78edd5827829 comparison_evaluations.jsonl
5
+ 3cd6ef87aaffad63379fff9034e9b11338c4fe6fc7c08f02f042bcb11d03fc3f convergence.json
6
+ f3f60c421d7daf57b52386a926e21090bc5dd19eaf7a58a4ca119a3a2b21527e convergence.jsonl
7
+ c11af8b26b66ce4ebe4d0a7f7ae3fc93aaef13fd8ac5dbf5920b8318fb2c923a eval/final.parquet
8
+ 4ea25cb934169c5a5a7cd7931e4a7458d26d3570f1452006073d75e17f211760 final/source_metadata.json
9
+ bc96fb09cf2841fda4b9b5e2eca95950e4fe4a6bbcfe22862975d2a25638a494 final/source_resolved_config.json
10
+ 70dda4b58013596119a7b3bbb3d85a2f871fba438451a99f2754c18290928ead final/state.pkl
11
+ 5a301a94981b7262ee403f3992b97df91c4c5f76039f443b817ec65570eff00f frozen_base.json
12
+ b2de1b68ed666ff92ed9642188dab2852338b5beda2b68d1f9c5438dae9ba1b1 offline_buffer.json
13
+ a5019e18225cb5b48e71b79460a549ef007aa39ddc16c069700926758589c421 publication.json
14
+ bc96fb09cf2841fda4b9b5e2eca95950e4fe4a6bbcfe22862975d2a25638a494 resolved_config.json
15
+ 3b329e445f47875515e3c7a917022a705696c369a18eb4852b1f1d92fae0f953 start_protocol.json
16
+ ebf44b4cda95c53187609b4d50e298b811d1c0234a6c53ee315a91da3e1714c9 status.json
17
+ bafeafda72884e98ea01e5bfcd3bbd92ec325c4ee05f0cd1ee8da2bdce423cff training_accounting.json
RL/ToolHang100/QC/mixed100/seed_1/best/source_metadata.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_sha256": "6ddafc0ff69f6fdea7c2951cad4b6ba09f3e32375fd411f9cd7abbec81adb8d9",
3
+ "evaluation": {
4
+ "chunk_transitions": 18000,
5
+ "episodes": 200,
6
+ "eval_index": 7,
7
+ "gain": null,
8
+ "gradient_updates": 54000,
9
+ "ma": null,
10
+ "n_success": 66,
11
+ "policy_version": 18,
12
+ "rl_env_steps": 143879,
13
+ "round": 18,
14
+ "seed_start": 960000,
15
+ "sr": 0.33,
16
+ "success_requested": 0,
17
+ "success_requested_imitation": 0,
18
+ "success_rounds_with_skips": 0,
19
+ "success_skipped": 0,
20
+ "success_skipped_imitation": 0,
21
+ "success_skipped_td": 0,
22
+ "target_step": 140000,
23
+ "update_index": 54000
24
+ },
25
+ "includes": "complete learner state, optimizer, targets, RNG and replay",
26
+ "selection": "observed_periodic_sr"
27
+ }
RL/ToolHang100/QC/mixed100/seed_1/best/source_resolved_config.json ADDED
@@ -0,0 +1,1195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
14
+ "key": "n_step_chunks",
15
+ "reason": "task-specific unified Study2 setting",
16
+ "upstream": 3,
17
+ "value": 5
18
+ },
19
+ {
20
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
21
+ "key": "offline.cohort_size",
22
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
23
+ "upstream": 50,
24
+ "value": 100
25
+ },
26
+ {
27
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
28
+ "key": "offline.episodes",
29
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
30
+ "upstream": 50,
31
+ "value": 100
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.act_bounds_json",
36
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
37
+ "upstream": null,
38
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}"
39
+ },
40
+ {
41
+ "decision_ref": "[0930-S2DEC]",
42
+ "key": "learner.qc.agent.act_max",
43
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
44
+ "upstream": 1.0,
45
+ "value": 3.0794031620025635
46
+ },
47
+ {
48
+ "decision_ref": "[0930-S2DEC]",
49
+ "key": "learner.qc.agent.act_min",
50
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
51
+ "upstream": -1.0,
52
+ "value": -3.0920298099517822
53
+ },
54
+ {
55
+ "decision_ref": "[0930-S2DEC]",
56
+ "key": "learner.qc.agent.actor_lr_partition",
57
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
58
+ "upstream": "lr=0.0003 shared by actor and critic",
59
+ "value": 0.0001
60
+ },
61
+ {
62
+ "decision_ref": "[1002-S2ALIGN]",
63
+ "key": "learner.qc.agent.actor_num_samples",
64
+ "reason": "single BoN knob including QC TD",
65
+ "upstream": 32,
66
+ "value": 16
67
+ },
68
+ {
69
+ "decision_ref": "[1002-S2ALIGN]",
70
+ "key": "learner.qc.agent.best_of_n",
71
+ "reason": "single BoN knob: collection/eval/QC TD",
72
+ "upstream": 8,
73
+ "value": 16
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.bon_agg",
78
+ "reason": "Best-of-N selection by the ensemble mean",
79
+ "upstream": null,
80
+ "value": "mean"
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.bon_sft",
85
+ "reason": "variant qc flag bon_sft",
86
+ "upstream": "upstream_yaml",
87
+ "value": false
88
+ },
89
+ {
90
+ "decision_ref": "[0930-S2DEC]",
91
+ "key": "learner.qc.agent.critic_lr_partition",
92
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
93
+ "upstream": "lr=0.0003 shared by actor and critic",
94
+ "value": 0.0003
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.partition_optimizer",
99
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
100
+ "upstream": null,
101
+ "value": true
102
+ },
103
+ {
104
+ "decision_ref": "docs/scope.md [0928-I]",
105
+ "key": "learner.qc.agent.q_target_agg",
106
+ "reason": "REDQ-style TD target: min over a random critic subset",
107
+ "upstream": null,
108
+ "value": "subset_min"
109
+ },
110
+ {
111
+ "decision_ref": "docs/scope.md [0928-I]",
112
+ "key": "learner.qc.agent.q_target_replace",
113
+ "reason": "subset drawn without replacement",
114
+ "upstream": null,
115
+ "value": false
116
+ },
117
+ {
118
+ "decision_ref": "docs/scope.md [0928-I]",
119
+ "key": "learner.qc.agent.q_target_subset",
120
+ "reason": "TD-target subset size",
121
+ "upstream": null,
122
+ "value": 2
123
+ },
124
+ {
125
+ "decision_ref": "[0930-S2DEC]",
126
+ "key": "learner.qc.agent.tau",
127
+ "reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
128
+ "upstream": 0.05,
129
+ "value": 0.005
130
+ },
131
+ {
132
+ "decision_ref": "[0930-S2DEC]",
133
+ "key": "learner.qc.agent.value_hidden_dims",
134
+ "reason": "unified.critic_hidden (critics only; actors native)",
135
+ "upstream": [
136
+ 512,
137
+ 512,
138
+ 512,
139
+ 512,
140
+ 512
141
+ ],
142
+ "value": [
143
+ 512,
144
+ 512,
145
+ 512
146
+ ]
147
+ },
148
+ {
149
+ "decision_ref": "docs/scope.md [0928-I]",
150
+ "key": "learner.qc.training.bc_pi_steps",
151
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
152
+ "upstream": 1000000,
153
+ "value": 0
154
+ },
155
+ {
156
+ "decision_ref": "docs/scope.md [0928-I]",
157
+ "key": "learner.qc.loop.buffer.utd_online",
158
+ "reason": "UTD=3 per executed action chunk",
159
+ "upstream": 1,
160
+ "value": "3 gradient updates per executed chunk"
161
+ },
162
+ {
163
+ "decision_ref": "docs/scope.md [0928-I]",
164
+ "key": "learner.qc.loop.n_step",
165
+ "reason": "n-step=3 counted in action chunks",
166
+ "upstream": "1 chunk (sequence_length = horizon_length)",
167
+ "value": "5 executed chunks (unified_env_step window)"
168
+ },
169
+ {
170
+ "decision_ref": "docs/scope.md [0928-I]",
171
+ "key": "learner.qc.loop.critic_actor_ratio",
172
+ "reason": "critic every update, actor every 2nd global update",
173
+ "upstream": "1:1 (utd_q = utd_pi)",
174
+ "value": "2:1"
175
+ },
176
+ {
177
+ "decision_ref": "docs/scope.md [0928-I]",
178
+ "key": "learner.qc.loop.buffer.start_training",
179
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
180
+ "upstream": 10000,
181
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
182
+ },
183
+ {
184
+ "decision_ref": "docs/scope.md [0928-I]",
185
+ "key": "learner.qc.loop.buffer.offline_ratio",
186
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
187
+ "upstream": 0.0,
188
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
189
+ },
190
+ {
191
+ "decision_ref": "docs/scope.md [0928-I]",
192
+ "key": "learner.qc.loop.buffer.size",
193
+ "reason": "one shared replay of executed chunks; success buffer = mask",
194
+ "upstream": 2000000,
195
+ "value": "1000000 executed chunks, shared FIFO"
196
+ },
197
+ {
198
+ "decision_ref": "docs/scope.md [0928-I]",
199
+ "key": "learner.qc.actor_architecture",
200
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
201
+ "upstream": [
202
+ 512,
203
+ 512,
204
+ 512,
205
+ 512,
206
+ 512
207
+ ],
208
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
209
+ },
210
+ {
211
+ "decision_ref": "docs/scope.md [0928-I]",
212
+ "key": "learner.qc.observation",
213
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
214
+ "upstream": "low-dim state",
215
+ "value": "frozen Ri features c [137] from the checkpoint encoder"
216
+ },
217
+ {
218
+ "decision_ref": "[0930-S2DEC]",
219
+ "key": "learner.qc.reward",
220
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
221
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
222
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
223
+ },
224
+ {
225
+ "decision_ref": "[0930-S2DEC]",
226
+ "key": "learner.qc.td_target",
227
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
228
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
229
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
230
+ },
231
+ {
232
+ "decision_ref": "[0930-S2DEC]",
233
+ "key": "learner.qc.eval",
234
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
235
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
236
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
237
+ },
238
+ {
239
+ "decision_ref": "[1002-S2ALIGN]",
240
+ "key": "learner.qc.lr_schedule",
241
+ "reason": "RL env-step clock shared with extra success TD",
242
+ "upstream": "native per-optimizer schedule",
243
+ "value": {
244
+ "actor": 0.0001,
245
+ "critic": 0.0003,
246
+ "decay_end_env_steps": 1606196,
247
+ "min_ratio": 0.1,
248
+ "schedule": "warmup_cosine",
249
+ "temperature": 0.0003,
250
+ "warmup_env_steps": 20000
251
+ }
252
+ },
253
+ {
254
+ "decision_ref": "[0930-S2DEC]",
255
+ "key": "learner.qc.agent.q_min_q_max",
256
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
257
+ "upstream": [
258
+ -999.9999999999991,
259
+ 0.0
260
+ ],
261
+ "value": [
262
+ 0.0,
263
+ 1.0
264
+ ]
265
+ }
266
+ ],
267
+ "protocol": {
268
+ "checkpoint_every_rounds": 5,
269
+ "checkpoint_retention": {
270
+ "best": true,
271
+ "collapse_drop": {
272
+ "min_abs": 0.2,
273
+ "ratio": 0.5
274
+ },
275
+ "diagnostics": true,
276
+ "every_evals": 3
277
+ },
278
+ "collection": {
279
+ "n_envs": 4,
280
+ "train_seed_base": 100000000,
281
+ "train_seed_stride": 1000000,
282
+ "training_freq_chunks": 1000
283
+ },
284
+ "convergence": {
285
+ "early_stop": false,
286
+ "gain_lag": 10,
287
+ "high_sr": 0.95,
288
+ "high_sr_evals": 5,
289
+ "ma_window": 10,
290
+ "max_env_steps": 1606196,
291
+ "min_env_steps": 200000,
292
+ "min_gain": 0.02,
293
+ "zero_success_evals": 8
294
+ },
295
+ "critic_ensemble": {
296
+ "num": 10,
297
+ "policy": "mean",
298
+ "target": {
299
+ "kind": "subset_min",
300
+ "replace": false,
301
+ "size": 2
302
+ }
303
+ },
304
+ "eval": {
305
+ "best_of_n": 16,
306
+ "episodes": 200,
307
+ "every_env_steps": 20000,
308
+ "include_step0": true,
309
+ "seed_start": 960000,
310
+ "workers": 4
311
+ },
312
+ "final_eval": {
313
+ "episodes": 300,
314
+ "on_zero_stop": true,
315
+ "seed_start": 800000
316
+ },
317
+ "n_step_chunks": 5,
318
+ "offline": {
319
+ "adaptive_ratio": false,
320
+ "cohort_size": 100,
321
+ "episodes": 100,
322
+ "learners_without_offline": [
323
+ "dsrl_sac"
324
+ ],
325
+ "ratio": {
326
+ "decay_env_steps": {
327
+ "can": 160000,
328
+ "square": 320000,
329
+ "tool_hang": 640000,
330
+ "transport": 320000
331
+ },
332
+ "end": 0.1,
333
+ "start": {
334
+ "can": 0.7,
335
+ "square": 0.5,
336
+ "tool_hang": 0.5,
337
+ "transport": 0.9
338
+ }
339
+ },
340
+ "selection": "ascending_demo_index",
341
+ "source": "bc_regime",
342
+ "unsupported_learners": [
343
+ "zprl"
344
+ ]
345
+ },
346
+ "replay": {
347
+ "capacity_chunks": 1000000,
348
+ "pin_start_successes": false,
349
+ "success_buffer": "success_mask_on_shared_fifo",
350
+ "success_skip_rule": "skip_below_success_batch_size",
351
+ "unit": "executed_chunk"
352
+ },
353
+ "start": {
354
+ "max_episodes": 200,
355
+ "mode": "offline_buffer",
356
+ "policy": {
357
+ "default": "base_cfm_ode",
358
+ "zprl": "vib_base"
359
+ },
360
+ "target_successes": 20
361
+ },
362
+ "unified": {
363
+ "actor_target_tau": 0.005,
364
+ "critic_hidden": [
365
+ 512,
366
+ 512,
367
+ 512
368
+ ],
369
+ "critic_target_every_critic_step": true,
370
+ "critic_target_tau": 0.005,
371
+ "dsrl_noise_imitation_reference_dims": 28,
372
+ "gamma_env": 0.999,
373
+ "lr": {
374
+ "actor": 0.0001,
375
+ "critic": 0.0003,
376
+ "decay_end_env_steps": 1606196,
377
+ "min_ratio": 0.1,
378
+ "schedule": "warmup_cosine",
379
+ "temperature": 0.0003,
380
+ "warmup_env_steps": 20000
381
+ },
382
+ "reward": "binary_first_success",
383
+ "success_td_polyak": true,
384
+ "timeout_bootstrap": false,
385
+ "zprl_imitation": "mse_pretanh_mean"
386
+ },
387
+ "updates": {
388
+ "batch_size": 256,
389
+ "critic_per_actor": 2,
390
+ "success_batch_size": 256,
391
+ "utd_per_chunk": 3
392
+ },
393
+ "version": "study2_protocol_v10"
394
+ },
395
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
396
+ "provenance_extra": {
397
+ "agent_library": {
398
+ "framework": "jax",
399
+ "sources": {
400
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
401
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
402
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
403
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
404
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
405
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
406
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
407
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
408
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
409
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
410
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
411
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
412
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
413
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
414
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
415
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
416
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
417
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
418
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
419
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
420
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
421
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
422
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
423
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
424
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
425
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
426
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
427
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
428
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
429
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
430
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
431
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
432
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
433
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
434
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
435
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
436
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
437
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
438
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
439
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
440
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
441
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
442
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
443
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
444
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
445
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
446
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
447
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
448
+ "train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
449
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
450
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
451
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
452
+ },
453
+ "version": "study2_jax_agents_v2"
454
+ },
455
+ "base_framework": "jax",
456
+ "bc_identity": {
457
+ "architecture": "dice_vision_resnet_flow_unet_v1",
458
+ "family": "StandardBaseBC",
459
+ "framework": "jax"
460
+ },
461
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/base/mixed100/epoch_300.pkl",
462
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
463
+ "experiment": "tool_hang",
464
+ "frozen_hash": "848c66c08785dca4f652c3a7eb0bd2cb26761fbdf91fb1dbcc2cde5be0446931",
465
+ "learner": {
466
+ "action_bounds": {
467
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
468
+ "compat_act_max": 3.0794031620025635,
469
+ "compat_act_min": -3.0920298099517822,
470
+ "hi": [
471
+ 1.0,
472
+ 1.0,
473
+ 1.0,
474
+ 3.0794031620025635,
475
+ 1.9159542322158813,
476
+ 1.0,
477
+ 1.0
478
+ ],
479
+ "kind": "per_dimension",
480
+ "lo": [
481
+ -1.0,
482
+ -1.0,
483
+ -1.0,
484
+ -3.0920298099517822,
485
+ -2.0611963272094727,
486
+ -1.531522274017334,
487
+ -1.0
488
+ ],
489
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
490
+ },
491
+ "agent_config": {
492
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
493
+ "act_max": 3.0794031620025635,
494
+ "act_min": -3.0920298099517822,
495
+ "action_chunking": true,
496
+ "action_critic_hidden_dims": null,
497
+ "action_dim": 7,
498
+ "action_num_qs": 10,
499
+ "action_repeat": false,
500
+ "actor_backbone": "mlp",
501
+ "actor_decay_steps": 50000,
502
+ "actor_end_value": 2e-05,
503
+ "actor_hidden_dims": [
504
+ 512,
505
+ 512,
506
+ 512,
507
+ 512,
508
+ 512
509
+ ],
510
+ "actor_layer_norm": false,
511
+ "actor_lr": 0.0003,
512
+ "actor_lr_partition": 0.0001,
513
+ "actor_num_samples": 16,
514
+ "actor_obs": "state",
515
+ "actor_scheduler": "cosine",
516
+ "actor_tau": 0.005,
517
+ "actor_type": "best-of-n",
518
+ "actor_warmup_steps": 2000,
519
+ "actor_weight_decay": 0.0,
520
+ "adv_clip_min": null,
521
+ "agent_name": "acfql",
522
+ "alpha": 500,
523
+ "batch_size": 256,
524
+ "best_of_n": 16,
525
+ "bon_agg": "mean",
526
+ "bon_sft": false,
527
+ "bon_sft_coeff": 1.0,
528
+ "bootstrap_ratio": 0.5,
529
+ "bptt_num_samples": 4,
530
+ "calql_q_subsample": 2,
531
+ "clip_grad_norm": 1000.0,
532
+ "clip_intermediate_actions": true,
533
+ "clip_max_epsilon_multiplier": 1.0,
534
+ "clip_min_epsilon_multiplier": 1.0,
535
+ "constant_noise_std": 0.01,
536
+ "cql_alpha": 5.0,
537
+ "cql_n_actions": 10,
538
+ "cql_temp": 1.0,
539
+ "critic_backbone": "mlp",
540
+ "critic_decay_steps": 5000,
541
+ "critic_end_value": 1e-08,
542
+ "critic_loss_type": "mse",
543
+ "critic_lr": 0.0003,
544
+ "critic_lr_partition": 0.0003,
545
+ "critic_obs": "state",
546
+ "critic_scheduler": "constant",
547
+ "critic_warmup_steps": 500,
548
+ "critic_weight_decay": 1e-05,
549
+ "denoised_clip_value": 1.0,
550
+ "discount": 0.999,
551
+ "edit_action_scale": 0.5,
552
+ "edit_actor_hidden_dims": null,
553
+ "edit_hidden_dims": [
554
+ 256,
555
+ 256,
556
+ 256
557
+ ],
558
+ "edit_init_temperature": 1.0,
559
+ "edit_layer_norm": true,
560
+ "edit_lr": 0.0003,
561
+ "edit_policy_bound": 0.1,
562
+ "edit_start_step": 100000,
563
+ "edit_target_entropy": null,
564
+ "edit_weight_decay": 0.0,
565
+ "encoder": null,
566
+ "entropy_backup": false,
567
+ "flow_steps": 10,
568
+ "fql_distillation_coeff": 10.0,
569
+ "fql_train_main_policy": true,
570
+ "freeze_vision_encoder": false,
571
+ "ft_flow_steps": 10,
572
+ "horizon_length": 8,
573
+ "init_temperature": 1.0,
574
+ "layer_norm": true,
575
+ "lr": 0.0003,
576
+ "lr_action_critic": null,
577
+ "lr_actor": null,
578
+ "lr_actor_bc_flow": null,
579
+ "lr_critic": null,
580
+ "lr_edit_actor": null,
581
+ "lr_noise_critic": null,
582
+ "max_noise_std": 0.01,
583
+ "mc_regression": false,
584
+ "mc_regression_coeff": 1.0,
585
+ "min_noise_std": 0.01,
586
+ "mip_q_ensemble": false,
587
+ "mip_q_noise_scale": 1.0,
588
+ "mip_q_t_star": 0.9,
589
+ "mip_t_star": 0.999,
590
+ "muon_beta": 0.95,
591
+ "muon_nesterov": true,
592
+ "muon_ns_steps": 5,
593
+ "n_action_samples": 8,
594
+ "n_edit_samples": 8,
595
+ "noise_critic_grad_steps": 1,
596
+ "noise_dim": null,
597
+ "noise_net_hidden_dims": [
598
+ 256,
599
+ 256
600
+ ],
601
+ "noise_net_layer_norm": true,
602
+ "normalize_act_space_dimension": false,
603
+ "normalize_denoising_horizon": false,
604
+ "normalize_q_loss": false,
605
+ "num_bins": 256,
606
+ "num_ensemble_members": 10,
607
+ "num_qs": 10,
608
+ "ob_dims": [
609
+ 137
610
+ ],
611
+ "one_step_decay_steps": null,
612
+ "one_step_end_value": null,
613
+ "one_step_hidden_dims": [
614
+ 512,
615
+ 512,
616
+ 512,
617
+ 512
618
+ ],
619
+ "one_step_layer_norm": false,
620
+ "one_step_lr": null,
621
+ "one_step_scheduler": null,
622
+ "one_step_warmup_steps": null,
623
+ "optimizer_action_critic": null,
624
+ "optimizer_actor": "adam",
625
+ "optimizer_actor_bc_flow": "adam",
626
+ "optimizer_critic": "adam",
627
+ "optimizer_edit_actor": null,
628
+ "paligemma_tokenizer_path": null,
629
+ "partition_optimizer": true,
630
+ "policy_type": "flow",
631
+ "ppo_batch_size": 256,
632
+ "ppo_lr": 4.5e-05,
633
+ "proprio_dim": null,
634
+ "q_agg": "mean",
635
+ "q_max": 1.0,
636
+ "q_min": 0.0,
637
+ "q_target_agg": "subset_min",
638
+ "q_target_replace": false,
639
+ "q_target_subset": 2,
640
+ "q_variance_reduction": false,
641
+ "q_vr_num_samples": 8,
642
+ "q_vr_reduction": "mean",
643
+ "randn_clip_value": 3.0,
644
+ "shortcut_denoising_steps": 4,
645
+ "shortcut_inference_steps": 4,
646
+ "simba_actor_hidden_dims": [
647
+ 1024,
648
+ 1024
649
+ ],
650
+ "simba_critic_hidden_dims": [
651
+ 1024,
652
+ 1024,
653
+ 1024
654
+ ],
655
+ "subsample_bon": false,
656
+ "target_entropy": null,
657
+ "tau": 0.005,
658
+ "tf_pi_conditioning": "adaln",
659
+ "tf_pi_dropout": 0.0,
660
+ "tf_pi_embed_dim": 256,
661
+ "tf_pi_heads": 4,
662
+ "tf_pi_layers": 8,
663
+ "tf_q_conditioning": "adaln",
664
+ "tf_q_dropout": 0.0,
665
+ "tf_q_embed_dim": 256,
666
+ "tf_q_heads": 4,
667
+ "tf_q_layers": 8,
668
+ "time_embedding": "sinusoidal",
669
+ "time_embedding_dim": 32,
670
+ "train_one_step_in_offline": true,
671
+ "train_one_step_in_online": true,
672
+ "useFiLM": false,
673
+ "useSimBa": false,
674
+ "useTF": false,
675
+ "use_constant_noise": true,
676
+ "use_constant_scheduler_for_bc": true,
677
+ "use_denoiser": false,
678
+ "use_edit_actor": false,
679
+ "use_mip_q": false,
680
+ "use_muon": false,
681
+ "use_one_step_policy": false,
682
+ "use_shortcut": false,
683
+ "use_state": "proprio",
684
+ "use_success_buffer_q": false,
685
+ "value_hidden_dims": [
686
+ 512,
687
+ 512,
688
+ 512
689
+ ],
690
+ "vit_encoder": "minvit",
691
+ "vit_img_proj_dim": 0,
692
+ "vit_state_proj_dim": 0
693
+ },
694
+ "convention": "unified_env_step",
695
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
696
+ "gamma_env": 0.999,
697
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
698
+ "learner": "qc",
699
+ "optimizers": {
700
+ "actor_lr": 0.0001,
701
+ "critic_lr": 0.0003,
702
+ "reset_optimizers_with_lr": "not used",
703
+ "schedule": {
704
+ "actor": 0.0001,
705
+ "critic": 0.0003,
706
+ "decay_end_env_steps": 1606196,
707
+ "min_ratio": 0.1,
708
+ "schedule": "warmup_cosine",
709
+ "temperature": 0.0003,
710
+ "warmup_env_steps": 20000
711
+ }
712
+ },
713
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
714
+ "recipe": {
715
+ "derived": {
716
+ "agent.actor_num_samples": {
717
+ "after": 8,
718
+ "before": 32
719
+ },
720
+ "agent.actor_type": {
721
+ "after": "best-of-n",
722
+ "before": "upstream_yaml"
723
+ },
724
+ "agent.bon_sft": {
725
+ "after": false,
726
+ "before": "upstream_yaml"
727
+ }
728
+ },
729
+ "explicit_overrides": {},
730
+ "source": "third_party/ogpo/scripts/qc/toolhang.sh",
731
+ "upstream_exact": false
732
+ },
733
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
734
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
735
+ "scan_block": 100,
736
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
737
+ "success_terms": {
738
+ "bc_on_success_batch": false,
739
+ "critic_success_td": false,
740
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
741
+ },
742
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
743
+ "unified_notes": {
744
+ "critic_target_every_critic_step": {
745
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
746
+ "value": true
747
+ },
748
+ "success_td_polyak": {
749
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
750
+ "value": true
751
+ }
752
+ },
753
+ "upstream_config": {
754
+ "agent": {
755
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
756
+ "act_max": 3.0794031620025635,
757
+ "act_min": -3.0920298099517822,
758
+ "action_chunking": true,
759
+ "action_critic_hidden_dims": null,
760
+ "action_dim": null,
761
+ "action_num_qs": 10,
762
+ "action_repeat": false,
763
+ "actor_backbone": "mlp",
764
+ "actor_decay_steps": 50000,
765
+ "actor_end_value": 2e-05,
766
+ "actor_hidden_dims": [
767
+ 512,
768
+ 512,
769
+ 512,
770
+ 512,
771
+ 512
772
+ ],
773
+ "actor_layer_norm": false,
774
+ "actor_lr": 0.0003,
775
+ "actor_lr_partition": 0.0001,
776
+ "actor_num_samples": 16,
777
+ "actor_obs": "state",
778
+ "actor_scheduler": "cosine",
779
+ "actor_tau": 0.005,
780
+ "actor_type": "best-of-n",
781
+ "actor_warmup_steps": 2000,
782
+ "actor_weight_decay": 0.0,
783
+ "adv_clip_min": null,
784
+ "agent_name": "acfql",
785
+ "alpha": 500,
786
+ "batch_size": 256,
787
+ "best_of_n": 16,
788
+ "bon_agg": "mean",
789
+ "bon_sft": false,
790
+ "bon_sft_coeff": 1.0,
791
+ "bootstrap_ratio": 0.5,
792
+ "bptt_num_samples": 4,
793
+ "calql_q_subsample": 2,
794
+ "clip_grad_norm": 1000.0,
795
+ "clip_intermediate_actions": true,
796
+ "clip_max_epsilon_multiplier": 1.0,
797
+ "clip_min_epsilon_multiplier": 1.0,
798
+ "constant_noise_std": 0.01,
799
+ "cql_alpha": 5.0,
800
+ "cql_n_actions": 10,
801
+ "cql_temp": 1.0,
802
+ "critic_backbone": "mlp",
803
+ "critic_decay_steps": 5000,
804
+ "critic_end_value": 1e-08,
805
+ "critic_loss_type": "mse",
806
+ "critic_lr": 0.0003,
807
+ "critic_lr_partition": 0.0003,
808
+ "critic_obs": "state",
809
+ "critic_scheduler": "constant",
810
+ "critic_warmup_steps": 500,
811
+ "critic_weight_decay": 1e-05,
812
+ "denoised_clip_value": 1.0,
813
+ "discount": 0.999,
814
+ "edit_action_scale": 0.5,
815
+ "edit_actor_hidden_dims": null,
816
+ "edit_hidden_dims": [
817
+ 256,
818
+ 256,
819
+ 256
820
+ ],
821
+ "edit_init_temperature": 1.0,
822
+ "edit_layer_norm": true,
823
+ "edit_lr": 0.0003,
824
+ "edit_policy_bound": 0.1,
825
+ "edit_start_step": 100000,
826
+ "edit_target_entropy": null,
827
+ "edit_weight_decay": 0.0,
828
+ "encoder": null,
829
+ "entropy_backup": false,
830
+ "flow_steps": 10,
831
+ "fql_distillation_coeff": 10.0,
832
+ "fql_train_main_policy": true,
833
+ "freeze_vision_encoder": false,
834
+ "ft_flow_steps": 10,
835
+ "horizon_length": 8,
836
+ "init_temperature": 1.0,
837
+ "layer_norm": true,
838
+ "lr": 0.0003,
839
+ "lr_action_critic": null,
840
+ "lr_actor": null,
841
+ "lr_actor_bc_flow": null,
842
+ "lr_critic": null,
843
+ "lr_edit_actor": null,
844
+ "lr_noise_critic": null,
845
+ "max_noise_std": 0.01,
846
+ "mc_regression": false,
847
+ "mc_regression_coeff": 1.0,
848
+ "min_noise_std": 0.01,
849
+ "mip_q_ensemble": false,
850
+ "mip_q_noise_scale": 1.0,
851
+ "mip_q_t_star": 0.9,
852
+ "mip_t_star": 0.999,
853
+ "muon_beta": 0.95,
854
+ "muon_nesterov": true,
855
+ "muon_ns_steps": 5,
856
+ "n_action_samples": 8,
857
+ "n_edit_samples": 8,
858
+ "noise_critic_grad_steps": 1,
859
+ "noise_dim": null,
860
+ "noise_net_hidden_dims": [
861
+ 256,
862
+ 256
863
+ ],
864
+ "noise_net_layer_norm": true,
865
+ "normalize_act_space_dimension": false,
866
+ "normalize_denoising_horizon": false,
867
+ "normalize_q_loss": false,
868
+ "num_bins": 256,
869
+ "num_ensemble_members": 10,
870
+ "num_qs": 10,
871
+ "ob_dims": null,
872
+ "one_step_decay_steps": null,
873
+ "one_step_end_value": null,
874
+ "one_step_hidden_dims": [
875
+ 512,
876
+ 512,
877
+ 512,
878
+ 512
879
+ ],
880
+ "one_step_layer_norm": false,
881
+ "one_step_lr": null,
882
+ "one_step_scheduler": null,
883
+ "one_step_warmup_steps": null,
884
+ "optimizer_action_critic": null,
885
+ "optimizer_actor": "adam",
886
+ "optimizer_actor_bc_flow": "adam",
887
+ "optimizer_critic": "adam",
888
+ "optimizer_edit_actor": null,
889
+ "paligemma_tokenizer_path": null,
890
+ "partition_optimizer": true,
891
+ "policy_type": "flow",
892
+ "ppo_batch_size": 256,
893
+ "ppo_lr": 4.5e-05,
894
+ "proprio_dim": null,
895
+ "q_agg": "mean",
896
+ "q_max": null,
897
+ "q_min": null,
898
+ "q_target_agg": "subset_min",
899
+ "q_target_replace": false,
900
+ "q_target_subset": 2,
901
+ "q_variance_reduction": false,
902
+ "q_vr_num_samples": 8,
903
+ "q_vr_reduction": "mean",
904
+ "randn_clip_value": 3.0,
905
+ "shortcut_denoising_steps": 4,
906
+ "shortcut_inference_steps": 4,
907
+ "simba_actor_hidden_dims": [
908
+ 1024,
909
+ 1024
910
+ ],
911
+ "simba_critic_hidden_dims": [
912
+ 1024,
913
+ 1024,
914
+ 1024
915
+ ],
916
+ "subsample_bon": false,
917
+ "target_entropy": null,
918
+ "tau": 0.005,
919
+ "tf_pi_conditioning": "adaln",
920
+ "tf_pi_dropout": 0.0,
921
+ "tf_pi_embed_dim": 256,
922
+ "tf_pi_heads": 4,
923
+ "tf_pi_layers": 8,
924
+ "tf_q_conditioning": "adaln",
925
+ "tf_q_dropout": 0.0,
926
+ "tf_q_embed_dim": 256,
927
+ "tf_q_heads": 4,
928
+ "tf_q_layers": 8,
929
+ "time_embedding": "sinusoidal",
930
+ "time_embedding_dim": 32,
931
+ "train_one_step_in_offline": true,
932
+ "train_one_step_in_online": true,
933
+ "useFiLM": false,
934
+ "useSimBa": false,
935
+ "useTF": false,
936
+ "use_constant_noise": true,
937
+ "use_constant_scheduler_for_bc": true,
938
+ "use_denoiser": false,
939
+ "use_edit_actor": false,
940
+ "use_mip_q": false,
941
+ "use_muon": false,
942
+ "use_one_step_policy": false,
943
+ "use_shortcut": false,
944
+ "use_state": "proprio",
945
+ "use_success_buffer_q": false,
946
+ "value_hidden_dims": [
947
+ 512,
948
+ 512,
949
+ 512
950
+ ],
951
+ "vit_encoder": "minvit",
952
+ "vit_img_proj_dim": 0,
953
+ "vit_state_proj_dim": 0
954
+ },
955
+ "bc": {
956
+ "clip_bc": true,
957
+ "clip_bc_threshold": 0.45,
958
+ "clip_bc_wrt": "ODE"
959
+ },
960
+ "buffer": {
961
+ "offline_ratio": 0.0,
962
+ "size": 2000000,
963
+ "start_training": 10000,
964
+ "utd_online": 1,
965
+ "utd_pi": null,
966
+ "utd_q": null,
967
+ "utd_v": null,
968
+ "utd_warmup": 1
969
+ },
970
+ "checkpoint": {
971
+ "ep_resume": 0,
972
+ "restore_actor_path": null,
973
+ "restore_critic_path": null,
974
+ "restore_encoder_path": null
975
+ },
976
+ "dataset": {
977
+ "num_offline_trajs": -1,
978
+ "ogbench_dir": null,
979
+ "p_aug": 0.0,
980
+ "proportion": 1.0,
981
+ "replace_interval": 1000
982
+ },
983
+ "env": {
984
+ "name": "tool_hang-ph-low_dim",
985
+ "post_success_steps": 0,
986
+ "rew_fn": "sparse",
987
+ "sparse": false
988
+ },
989
+ "eval": {
990
+ "eval_episodes": 2,
991
+ "n_eval_envs": 32,
992
+ "plot_q_vs_mc": false,
993
+ "video_episodes": 0,
994
+ "video_frame_skip": 3
995
+ },
996
+ "experiment": {
997
+ "log": true,
998
+ "project": "QC",
999
+ "run_group": "toolhang",
1000
+ "run_name": "qc_toolhang",
1001
+ "save_dir": "exp/ogpo",
1002
+ "seed": 1,
1003
+ "wandb_name": "qc_toolhang"
1004
+ },
1005
+ "intervals": {
1006
+ "eval_interval": 20000,
1007
+ "eval_interval_bc": 5000,
1008
+ "log_interval": 5000,
1009
+ "preemption_backup_interval": -1,
1010
+ "save_interval": 500000,
1011
+ "save_interval_bc": -1,
1012
+ "step_data_log_interval": -1
1013
+ },
1014
+ "success_buffer": {
1015
+ "batch_size": 256,
1016
+ "use_success_buffer": false
1017
+ },
1018
+ "training": {
1019
+ "bc_only": false,
1020
+ "bc_pi_steps": 0,
1021
+ "bc_q_steps": 0,
1022
+ "bc_refine_steps": 0,
1023
+ "calql_steps": 0,
1024
+ "force_jax_sync": true,
1025
+ "gpu_mode": "single",
1026
+ "online_steps": 5000000,
1027
+ "q_warmup_steps": 0
1028
+ }
1029
+ },
1030
+ "upstream_source": {
1031
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1032
+ "patches_sha256": {
1033
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1034
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1035
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1036
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1037
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1038
+ }
1039
+ }
1040
+ },
1041
+ "observation_mode": "pixel_state",
1042
+ "offline_buffer": {
1043
+ "actions_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1044
+ "aux_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1045
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
1046
+ "chunks": 5946,
1047
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1048
+ "decision_ref": "[1017-S2OFFBC]",
1049
+ "env_steps": 47168,
1050
+ "features_sha256": "7af4a3aee67b4a7eae0c45d07974462977e71712cd76bb9afd89dfdb9130699b",
1051
+ "horizon": 8,
1052
+ "identity_hash": "d54f7fa39190225c48c52a3782ea8d00b6d59a0a08d5c6b085b1a6af164eff34",
1053
+ "learner_coordinates": "JaxStudy2Learner:qc",
1054
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1055
+ "ratio": {
1056
+ "adaptive_ratio": false,
1057
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1058
+ },
1059
+ "selection": {
1060
+ "cohort_size": 100,
1061
+ "demos": [
1062
+ "demo_0",
1063
+ "demo_5",
1064
+ "demo_6",
1065
+ "demo_7",
1066
+ "demo_9",
1067
+ "demo_12",
1068
+ "demo_16",
1069
+ "demo_17",
1070
+ "demo_18",
1071
+ "demo_20",
1072
+ "demo_25",
1073
+ "demo_27",
1074
+ "demo_30",
1075
+ "demo_31",
1076
+ "demo_32",
1077
+ "demo_35",
1078
+ "demo_37",
1079
+ "demo_38",
1080
+ "demo_43",
1081
+ "demo_44",
1082
+ "demo_45",
1083
+ "demo_46",
1084
+ "demo_50",
1085
+ "demo_51",
1086
+ "demo_55",
1087
+ "demo_56",
1088
+ "demo_57",
1089
+ "demo_58",
1090
+ "demo_69",
1091
+ "demo_70",
1092
+ "demo_71",
1093
+ "demo_72",
1094
+ "demo_74",
1095
+ "demo_78",
1096
+ "demo_79",
1097
+ "demo_81",
1098
+ "demo_82",
1099
+ "demo_84",
1100
+ "demo_85",
1101
+ "demo_86",
1102
+ "demo_90",
1103
+ "demo_91",
1104
+ "demo_92",
1105
+ "demo_95",
1106
+ "demo_96",
1107
+ "demo_97",
1108
+ "demo_98",
1109
+ "demo_101",
1110
+ "demo_104",
1111
+ "demo_107",
1112
+ "demo_111",
1113
+ "demo_112",
1114
+ "demo_113",
1115
+ "demo_115",
1116
+ "demo_116",
1117
+ "demo_119",
1118
+ "demo_121",
1119
+ "demo_123",
1120
+ "demo_125",
1121
+ "demo_127",
1122
+ "demo_128",
1123
+ "demo_131",
1124
+ "demo_132",
1125
+ "demo_133",
1126
+ "demo_134",
1127
+ "demo_137",
1128
+ "demo_138",
1129
+ "demo_140",
1130
+ "demo_141",
1131
+ "demo_142",
1132
+ "demo_145",
1133
+ "demo_149",
1134
+ "demo_150",
1135
+ "demo_151",
1136
+ "demo_154",
1137
+ "demo_157",
1138
+ "demo_159",
1139
+ "demo_161",
1140
+ "demo_164",
1141
+ "demo_165",
1142
+ "demo_169",
1143
+ "demo_171",
1144
+ "demo_172",
1145
+ "demo_173",
1146
+ "demo_175",
1147
+ "demo_176",
1148
+ "demo_177",
1149
+ "demo_178",
1150
+ "demo_179",
1151
+ "demo_181",
1152
+ "demo_182",
1153
+ "demo_183",
1154
+ "demo_184",
1155
+ "demo_185",
1156
+ "demo_187",
1157
+ "demo_188",
1158
+ "demo_191",
1159
+ "demo_192",
1160
+ "demo_193",
1161
+ "demo_196"
1162
+ ],
1163
+ "manifest": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/source_input/data_manifest.json",
1164
+ "manifest_sha256": "6bcfaec343fc97c766bacf650dc6a73a71c0c9969974d21bfbfd1c1ac113685f",
1165
+ "regime": "mixed100",
1166
+ "source": "bc_regime",
1167
+ "source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1168
+ "split": {
1169
+ "training": 90,
1170
+ "validation": 10
1171
+ },
1172
+ "task": "tool_hang"
1173
+ }
1174
+ },
1175
+ "offline_role": "mixed",
1176
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1177
+ "smoke": false,
1178
+ "task": "tool_hang",
1179
+ "task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
1180
+ },
1181
+ "seed": 1,
1182
+ "start_policy": "base_cfm_ode",
1183
+ "variant": {
1184
+ "flags": {
1185
+ "bon_sft": false,
1186
+ "use_success_buffer": false
1187
+ },
1188
+ "learner": "qc",
1189
+ "name": "qc",
1190
+ "phase": 1,
1191
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1192
+ "version": "study2_variants_v4"
1193
+ },
1194
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1195
+ }
RL/ToolHang100/QC/mixed100/seed_1/best/state.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ddafc0ff69f6fdea7c2951cad4b6ba09f3e32375fd411f9cd7abbec81adb8d9
3
+ size 207205195
RL/ToolHang100/QC/mixed100/seed_1/comparison_evaluations.jsonl ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"eval_index": 0, "rl_env_steps": 0, "target_step": 0, "sr": 0.165, "round": 0, "update_index": 0, "policy_version": 0, "chunk_transitions": 0, "gradient_updates": 0, "n_success": 33, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
2
+ {"eval_index": 1, "rl_env_steps": 23974, "target_step": 20000, "sr": 0.205, "round": 3, "update_index": 9000, "policy_version": 3, "chunk_transitions": 3000, "gradient_updates": 9000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
3
+ {"eval_index": 2, "rl_env_steps": 47941, "target_step": 40000, "sr": 0.17, "round": 6, "update_index": 18000, "policy_version": 6, "chunk_transitions": 6000, "gradient_updates": 18000, "n_success": 34, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
4
+ {"eval_index": 3, "rl_env_steps": 63941, "target_step": 60000, "sr": 0.31, "round": 8, "update_index": 24000, "policy_version": 8, "chunk_transitions": 8000, "gradient_updates": 24000, "n_success": 62, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
5
+ {"eval_index": 4, "rl_env_steps": 95912, "target_step": 80000, "sr": 0.21, "round": 12, "update_index": 36000, "policy_version": 12, "chunk_transitions": 12000, "gradient_updates": 36000, "n_success": 42, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
6
+ {"eval_index": 5, "rl_env_steps": 103912, "target_step": 100000, "sr": 0.285, "round": 13, "update_index": 39000, "policy_version": 13, "chunk_transitions": 13000, "gradient_updates": 39000, "n_success": 57, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
7
+ {"eval_index": 6, "rl_env_steps": 135890, "target_step": 120000, "sr": 0.295, "round": 17, "update_index": 51000, "policy_version": 17, "chunk_transitions": 17000, "gradient_updates": 51000, "n_success": 59, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
8
+ {"eval_index": 7, "rl_env_steps": 143879, "target_step": 140000, "sr": 0.33, "round": 18, "update_index": 54000, "policy_version": 18, "chunk_transitions": 18000, "gradient_updates": 54000, "n_success": 66, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
9
+ {"eval_index": 8, "rl_env_steps": 167846, "target_step": 160000, "sr": 0.32, "round": 21, "update_index": 63000, "policy_version": 21, "chunk_transitions": 21000, "gradient_updates": 63000, "n_success": 64, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
10
+ {"eval_index": 9, "rl_env_steps": 183840, "target_step": 180000, "sr": 0.275, "round": 23, "update_index": 69000, "policy_version": 23, "chunk_transitions": 23000, "gradient_updates": 69000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2565, "gain": null, "observed": true, "imputed": false}
11
+ {"eval_index": 10, "rl_env_steps": 207819, "target_step": 200000, "sr": 0.27, "round": 26, "update_index": 78000, "policy_version": 26, "chunk_transitions": 26000, "gradient_updates": 78000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.267, "gain": null, "observed": true, "imputed": false}
12
+ {"eval_index": 11, "rl_env_steps": 223806, "target_step": 220000, "sr": 0.295, "round": 28, "update_index": 84000, "policy_version": 28, "chunk_transitions": 28000, "gradient_updates": 84000, "n_success": 59, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.27599999999999997, "gain": null, "observed": true, "imputed": false}
13
+ {"eval_index": 12, "rl_env_steps": 247784, "target_step": 240000, "sr": 0.3, "round": 31, "update_index": 93000, "policy_version": 31, "chunk_transitions": 31000, "gradient_updates": 93000, "n_success": 60, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.289, "gain": null, "observed": true, "imputed": false}
14
+ {"eval_index": 13, "rl_env_steps": 263759, "target_step": 260000, "sr": 0.24, "round": 33, "update_index": 99000, "policy_version": 33, "chunk_transitions": 33000, "gradient_updates": 99000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.28200000000000003, "gain": null, "observed": true, "imputed": false}
15
+ {"eval_index": 14, "rl_env_steps": 287728, "target_step": 280000, "sr": 0.21, "round": 36, "update_index": 108000, "policy_version": 36, "chunk_transitions": 36000, "gradient_updates": 108000, "n_success": 42, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.28200000000000003, "gain": null, "observed": true, "imputed": false}
16
+ {"eval_index": 15, "rl_env_steps": 303728, "target_step": 300000, "sr": 0.255, "round": 38, "update_index": 114000, "policy_version": 38, "chunk_transitions": 38000, "gradient_updates": 114000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.279, "gain": null, "observed": true, "imputed": false}
17
+ {"eval_index": 16, "rl_env_steps": 327708, "target_step": 320000, "sr": 0.275, "round": 41, "update_index": 123000, "policy_version": 41, "chunk_transitions": 41000, "gradient_updates": 123000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.277, "gain": null, "observed": true, "imputed": false}
18
+ {"eval_index": 17, "rl_env_steps": 343708, "target_step": 340000, "sr": 0.22, "round": 43, "update_index": 129000, "policy_version": 43, "chunk_transitions": 43000, "gradient_updates": 129000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.266, "gain": null, "observed": true, "imputed": false}
19
+ {"eval_index": 18, "rl_env_steps": 367678, "target_step": 360000, "sr": 0.225, "round": 46, "update_index": 138000, "policy_version": 46, "chunk_transitions": 46000, "gradient_updates": 138000, "n_success": 45, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25650000000000006, "gain": null, "observed": true, "imputed": false}
20
+ {"eval_index": 19, "rl_env_steps": 383661, "target_step": 380000, "sr": 0.235, "round": 48, "update_index": 144000, "policy_version": 48, "chunk_transitions": 48000, "gradient_updates": 144000, "n_success": 47, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2525, "gain": -0.0040000000000000036, "observed": true, "imputed": false}
21
+ {"eval_index": 20, "rl_env_steps": 407624, "target_step": 400000, "sr": 0.265, "round": 51, "update_index": 153000, "policy_version": 51, "chunk_transitions": 51000, "gradient_updates": 153000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.252, "gain": -0.015000000000000013, "observed": true, "imputed": false}
22
+ {"eval_index": 21, "rl_env_steps": 423609, "target_step": 420000, "sr": 0.215, "round": 53, "update_index": 159000, "policy_version": 53, "chunk_transitions": 53000, "gradient_updates": 159000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.244, "gain": -0.03199999999999997, "observed": true, "imputed": false}
23
+ {"eval_index": 22, "rl_env_steps": 447596, "target_step": 440000, "sr": 0.255, "round": 56, "update_index": 168000, "policy_version": 56, "chunk_transitions": 56000, "gradient_updates": 168000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2395, "gain": -0.04949999999999999, "observed": true, "imputed": false}
24
+ {"eval_index": 23, "rl_env_steps": 463587, "target_step": 460000, "sr": 0.235, "round": 58, "update_index": 174000, "policy_version": 58, "chunk_transitions": 58000, "gradient_updates": 174000, "n_success": 47, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23899999999999996, "gain": -0.043000000000000066, "observed": true, "imputed": false}
25
+ {"eval_index": 24, "rl_env_steps": 487565, "target_step": 480000, "sr": 0.2, "round": 61, "update_index": 183000, "policy_version": 61, "chunk_transitions": 61000, "gradient_updates": 183000, "n_success": 40, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23800000000000004, "gain": -0.043999999999999984, "observed": true, "imputed": false}
26
+ {"eval_index": 25, "rl_env_steps": 503550, "target_step": 500000, "sr": 0.215, "round": 63, "update_index": 189000, "policy_version": 63, "chunk_transitions": 63000, "gradient_updates": 189000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23399999999999999, "gain": -0.04500000000000004, "observed": true, "imputed": false}
27
+ {"eval_index": 26, "rl_env_steps": 527543, "target_step": 520000, "sr": 0.27, "round": 66, "update_index": 198000, "policy_version": 66, "chunk_transitions": 66000, "gradient_updates": 198000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23349999999999999, "gain": -0.04350000000000004, "observed": true, "imputed": false}
28
+ {"eval_index": 27, "rl_env_steps": 543525, "target_step": 540000, "sr": 0.24, "round": 68, "update_index": 204000, "policy_version": 68, "chunk_transitions": 68000, "gradient_updates": 204000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23550000000000004, "gain": -0.03049999999999997, "observed": true, "imputed": false}
29
+ {"eval_index": 28, "rl_env_steps": 567508, "target_step": 560000, "sr": 0.325, "round": 71, "update_index": 213000, "policy_version": 71, "chunk_transitions": 71000, "gradient_updates": 213000, "n_success": 65, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2455, "gain": -0.011000000000000065, "observed": true, "imputed": false}
30
+ {"eval_index": 29, "rl_env_steps": 583487, "target_step": 580000, "sr": 0.245, "round": 73, "update_index": 219000, "policy_version": 73, "chunk_transitions": 73000, "gradient_updates": 219000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24650000000000002, "gain": -0.005999999999999978, "observed": true, "imputed": false}
31
+ {"eval_index": 30, "rl_env_steps": 607480, "target_step": 600000, "sr": 0.28, "round": 76, "update_index": 228000, "policy_version": 76, "chunk_transitions": 76000, "gradient_updates": 228000, "n_success": 56, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24799999999999994, "gain": -0.004000000000000059, "observed": true, "imputed": false}
32
+ {"eval_index": 31, "rl_env_steps": 623463, "target_step": 620000, "sr": 0.24, "round": 78, "update_index": 234000, "policy_version": 78, "chunk_transitions": 78000, "gradient_updates": 234000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2505, "gain": 0.006500000000000006, "observed": true, "imputed": false}
33
+ {"eval_index": 32, "rl_env_steps": 647437, "target_step": 640000, "sr": 0.25, "round": 81, "update_index": 243000, "policy_version": 81, "chunk_transitions": 81000, "gradient_updates": 243000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25, "gain": 0.01050000000000001, "observed": true, "imputed": false}
34
+ {"eval_index": 33, "rl_env_steps": 663425, "target_step": 660000, "sr": 0.17, "round": 83, "update_index": 249000, "policy_version": 83, "chunk_transitions": 83000, "gradient_updates": 249000, "n_success": 34, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": 0.004500000000000032, "observed": true, "imputed": false}
35
+ {"eval_index": 34, "rl_env_steps": 687396, "target_step": 680000, "sr": 0.25, "round": 86, "update_index": 258000, "policy_version": 86, "chunk_transitions": 86000, "gradient_updates": 258000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2485, "gain": 0.010499999999999954, "observed": true, "imputed": false}
36
+ {"eval_index": 35, "rl_env_steps": 703373, "target_step": 700000, "sr": 0.225, "round": 88, "update_index": 264000, "policy_version": 88, "chunk_transitions": 88000, "gradient_updates": 264000, "n_success": 45, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2495, "gain": 0.015500000000000014, "observed": true, "imputed": false}
37
+ {"eval_index": 36, "rl_env_steps": 727364, "target_step": 720000, "sr": 0.265, "round": 91, "update_index": 273000, "policy_version": 91, "chunk_transitions": 91000, "gradient_updates": 273000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24900000000000003, "gain": 0.015500000000000042, "observed": true, "imputed": false}
38
+ {"eval_index": 37, "rl_env_steps": 743345, "target_step": 740000, "sr": 0.215, "round": 93, "update_index": 279000, "policy_version": 93, "chunk_transitions": 93000, "gradient_updates": 279000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2465, "gain": 0.010999999999999954, "observed": true, "imputed": false}
39
+ {"eval_index": 38, "rl_env_steps": 767331, "target_step": 760000, "sr": 0.275, "round": 96, "update_index": 288000, "policy_version": 96, "chunk_transitions": 96000, "gradient_updates": 288000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2415, "gain": -0.0040000000000000036, "observed": true, "imputed": false}
40
+ {"eval_index": 39, "rl_env_steps": 783313, "target_step": 780000, "sr": 0.285, "round": 98, "update_index": 294000, "policy_version": 98, "chunk_transitions": 98000, "gradient_updates": 294000, "n_success": 57, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2455, "gain": -0.0010000000000000286, "observed": true, "imputed": false}
41
+ {"eval_index": 40, "rl_env_steps": 807291, "target_step": 800000, "sr": 0.245, "round": 101, "update_index": 303000, "policy_version": 101, "chunk_transitions": 101000, "gradient_updates": 303000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24200000000000005, "gain": -0.005999999999999894, "observed": true, "imputed": false}
42
+ {"eval_index": 41, "rl_env_steps": 823274, "target_step": 820000, "sr": 0.255, "round": 103, "update_index": 309000, "policy_version": 103, "chunk_transitions": 103000, "gradient_updates": 309000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": -0.007000000000000006, "observed": true, "imputed": false}
43
+ {"eval_index": 42, "rl_env_steps": 847246, "target_step": 840000, "sr": 0.255, "round": 106, "update_index": 318000, "policy_version": 106, "chunk_transitions": 106000, "gradient_updates": 318000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.244, "gain": -0.006000000000000005, "observed": true, "imputed": false}
44
+ {"eval_index": 43, "rl_env_steps": 863227, "target_step": 860000, "sr": 0.205, "round": 108, "update_index": 324000, "policy_version": 108, "chunk_transitions": 108000, "gradient_updates": 324000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2475, "gain": 0.0040000000000000036, "observed": true, "imputed": false}
45
+ {"eval_index": 44, "rl_env_steps": 887199, "target_step": 880000, "sr": 0.255, "round": 111, "update_index": 333000, "policy_version": 111, "chunk_transitions": 111000, "gradient_updates": 333000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.248, "gain": -0.0005000000000000004, "observed": true, "imputed": false}
46
+ {"eval_index": 45, "rl_env_steps": 903199, "target_step": 900000, "sr": 0.195, "round": 113, "update_index": 339000, "policy_version": 113, "chunk_transitions": 113000, "gradient_updates": 339000, "n_success": 39, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24499999999999997, "gain": -0.004500000000000032, "observed": true, "imputed": false}
47
+ {"eval_index": 46, "rl_env_steps": 927186, "target_step": 920000, "sr": 0.19, "round": 116, "update_index": 348000, "policy_version": 116, "chunk_transitions": 116000, "gradient_updates": 348000, "n_success": 38, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2375, "gain": -0.011500000000000038, "observed": true, "imputed": false}
48
+ {"eval_index": 47, "rl_env_steps": 943184, "target_step": 940000, "sr": 0.275, "round": 118, "update_index": 354000, "policy_version": 118, "chunk_transitions": 118000, "gradient_updates": 354000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": -0.0030000000000000027, "observed": true, "imputed": false}
49
+ {"eval_index": 48, "rl_env_steps": 967152, "target_step": 960000, "sr": 0.235, "round": 121, "update_index": 363000, "policy_version": 121, "chunk_transitions": 121000, "gradient_updates": 363000, "n_success": 47, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2395, "gain": -0.0020000000000000018, "observed": true, "imputed": false}
50
+ {"eval_index": 49, "rl_env_steps": 983141, "target_step": 980000, "sr": 0.275, "round": 123, "update_index": 369000, "policy_version": 123, "chunk_transitions": 123000, "gradient_updates": 369000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2385, "gain": -0.007000000000000006, "observed": true, "imputed": false}
51
+ {"eval_index": 50, "rl_env_steps": 1007110, "target_step": 1000000, "sr": 0.26, "round": 126, "update_index": 378000, "policy_version": 126, "chunk_transitions": 126000, "gradient_updates": 378000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24000000000000005, "gain": -0.0020000000000000018, "observed": true, "imputed": false}
52
+ {"eval_index": 51, "rl_env_steps": 1023084, "target_step": 1020000, "sr": 0.26, "round": 128, "update_index": 384000, "policy_version": 128, "chunk_transitions": 128000, "gradient_updates": 384000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24050000000000002, "gain": -0.002999999999999975, "observed": true, "imputed": false}
53
+ {"eval_index": 52, "rl_env_steps": 1047060, "target_step": 1040000, "sr": 0.24, "round": 131, "update_index": 393000, "policy_version": 131, "chunk_transitions": 131000, "gradient_updates": 393000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23899999999999996, "gain": -0.005000000000000032, "observed": true, "imputed": false}
54
+ {"eval_index": 53, "rl_env_steps": 1063023, "target_step": 1060000, "sr": 0.25, "round": 133, "update_index": 399000, "policy_version": 133, "chunk_transitions": 133000, "gradient_updates": 399000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": -0.0040000000000000036, "observed": true, "imputed": false}
55
+ {"eval_index": 54, "rl_env_steps": 1087011, "target_step": 1080000, "sr": 0.26, "round": 136, "update_index": 408000, "policy_version": 136, "chunk_transitions": 136000, "gradient_updates": 408000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24400000000000005, "gain": -0.003999999999999948, "observed": true, "imputed": false}
56
+ {"eval_index": 55, "rl_env_steps": 1102994, "target_step": 1100000, "sr": 0.29, "round": 138, "update_index": 414000, "policy_version": 138, "chunk_transitions": 138000, "gradient_updates": 414000, "n_success": 58, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2535, "gain": 0.008500000000000035, "observed": true, "imputed": false}
57
+ {"eval_index": 56, "rl_env_steps": 1126981, "target_step": 1120000, "sr": 0.24, "round": 141, "update_index": 423000, "policy_version": 141, "chunk_transitions": 141000, "gradient_updates": 423000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2585, "gain": 0.02100000000000002, "observed": true, "imputed": false}
58
+ {"eval_index": 57, "rl_env_steps": 1142956, "target_step": 1140000, "sr": 0.225, "round": 143, "update_index": 429000, "policy_version": 143, "chunk_transitions": 143000, "gradient_updates": 429000, "n_success": 45, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25350000000000006, "gain": 0.010000000000000064, "observed": true, "imputed": false}
59
+ {"eval_index": 58, "rl_env_steps": 1174936, "target_step": 1160000, "sr": 0.245, "round": 147, "update_index": 441000, "policy_version": 147, "chunk_transitions": 147000, "gradient_updates": 441000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25450000000000006, "gain": 0.015000000000000069, "observed": true, "imputed": false}
60
+ {"eval_index": 59, "rl_env_steps": 1182936, "target_step": 1180000, "sr": 0.23, "round": 148, "update_index": 444000, "policy_version": 148, "chunk_transitions": 148000, "gradient_updates": 444000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25000000000000006, "gain": 0.011500000000000066, "observed": true, "imputed": false}
61
+ {"eval_index": 60, "rl_env_steps": 1206913, "target_step": 1200000, "sr": 0.265, "round": 151, "update_index": 453000, "policy_version": 151, "chunk_transitions": 151000, "gradient_updates": 453000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2505, "gain": 0.010499999999999954, "observed": true, "imputed": false}
62
+ {"eval_index": 61, "rl_env_steps": 1222893, "target_step": 1220000, "sr": 0.19, "round": 153, "update_index": 459000, "policy_version": 153, "chunk_transitions": 153000, "gradient_updates": 459000, "n_success": 38, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": 0.002999999999999975, "observed": true, "imputed": false}
63
+ {"eval_index": 62, "rl_env_steps": 1246875, "target_step": 1240000, "sr": 0.22, "round": 156, "update_index": 468000, "policy_version": 156, "chunk_transitions": 156000, "gradient_updates": 468000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2415, "gain": 0.00250000000000003, "observed": true, "imputed": false}
64
+ {"eval_index": 63, "rl_env_steps": 1262860, "target_step": 1260000, "sr": 0.21, "round": 158, "update_index": 474000, "policy_version": 158, "chunk_transitions": 158000, "gradient_updates": 474000, "n_success": 42, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2375, "gain": -0.006000000000000005, "observed": true, "imputed": false}
65
+ {"eval_index": 64, "rl_env_steps": 1286833, "target_step": 1280000, "sr": 0.26, "round": 161, "update_index": 483000, "policy_version": 161, "chunk_transitions": 161000, "gradient_updates": 483000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2375, "gain": -0.006500000000000061, "observed": true, "imputed": false}
66
+ {"eval_index": 65, "rl_env_steps": 1302817, "target_step": 1300000, "sr": 0.285, "round": 163, "update_index": 489000, "policy_version": 163, "chunk_transitions": 163000, "gradient_updates": 489000, "n_success": 57, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23700000000000002, "gain": -0.016499999999999987, "observed": true, "imputed": false}
67
+ {"eval_index": 66, "rl_env_steps": 1326788, "target_step": 1320000, "sr": 0.2, "round": 166, "update_index": 498000, "policy_version": 166, "chunk_transitions": 166000, "gradient_updates": 498000, "n_success": 40, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.233, "gain": -0.025499999999999995, "observed": true, "imputed": false}
68
+ {"eval_index": 67, "rl_env_steps": 1342777, "target_step": 1340000, "sr": 0.185, "round": 168, "update_index": 504000, "policy_version": 168, "chunk_transitions": 168000, "gradient_updates": 504000, "n_success": 37, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.229, "gain": -0.02450000000000005, "observed": true, "imputed": false}
69
+ {"eval_index": 68, "rl_env_steps": 1366763, "target_step": 1360000, "sr": 0.275, "round": 171, "update_index": 513000, "policy_version": 171, "chunk_transitions": 171000, "gradient_updates": 513000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23199999999999998, "gain": -0.022500000000000075, "observed": true, "imputed": false}
70
+ {"eval_index": 69, "rl_env_steps": 1382739, "target_step": 1380000, "sr": 0.255, "round": 173, "update_index": 519000, "policy_version": 173, "chunk_transitions": 173000, "gradient_updates": 519000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2345, "gain": -0.01550000000000007, "observed": true, "imputed": false}
71
+ {"eval_index": 70, "rl_env_steps": 1406718, "target_step": 1400000, "sr": 0.215, "round": 176, "update_index": 528000, "policy_version": 176, "chunk_transitions": 176000, "gradient_updates": 528000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22949999999999998, "gain": -0.02100000000000002, "observed": true, "imputed": false}
72
+ {"eval_index": 71, "rl_env_steps": 1422707, "target_step": 1420000, "sr": 0.205, "round": 178, "update_index": 534000, "policy_version": 178, "chunk_transitions": 178000, "gradient_updates": 534000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.231, "gain": -0.012499999999999983, "observed": true, "imputed": false}
73
+ {"eval_index": 72, "rl_env_steps": 1446668, "target_step": 1440000, "sr": 0.265, "round": 181, "update_index": 543000, "policy_version": 181, "chunk_transitions": 181000, "gradient_updates": 543000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2355, "gain": -0.006000000000000005, "observed": true, "imputed": false}
74
+ {"eval_index": 73, "rl_env_steps": 1462642, "target_step": 1460000, "sr": 0.15, "round": 183, "update_index": 549000, "policy_version": 183, "chunk_transitions": 183000, "gradient_updates": 549000, "n_success": 30, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22949999999999998, "gain": -0.008000000000000007, "observed": true, "imputed": false}
75
+ {"eval_index": 74, "rl_env_steps": 1486630, "target_step": 1480000, "sr": 0.275, "round": 186, "update_index": 558000, "policy_version": 186, "chunk_transitions": 186000, "gradient_updates": 558000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.231, "gain": -0.006499999999999978, "observed": true, "imputed": false}
76
+ {"eval_index": 75, "rl_env_steps": 1502611, "target_step": 1500000, "sr": 0.27, "round": 188, "update_index": 564000, "policy_version": 188, "chunk_transitions": 188000, "gradient_updates": 564000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22949999999999998, "gain": -0.007500000000000034, "observed": true, "imputed": false}
77
+ {"eval_index": 76, "rl_env_steps": 1526589, "target_step": 1520000, "sr": 0.235, "round": 191, "update_index": 573000, "policy_version": 191, "chunk_transitions": 191000, "gradient_updates": 573000, "n_success": 47, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23299999999999996, "gain": -5.551115123125783e-17, "observed": true, "imputed": false}
78
+ {"eval_index": 77, "rl_env_steps": 1542576, "target_step": 1540000, "sr": 0.23, "round": 193, "update_index": 579000, "policy_version": 193, "chunk_transitions": 193000, "gradient_updates": 579000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2375, "gain": 0.00849999999999998, "observed": true, "imputed": false}
79
+ {"eval_index": 78, "rl_env_steps": 1566542, "target_step": 1560000, "sr": 0.225, "round": 196, "update_index": 588000, "policy_version": 196, "chunk_transitions": 196000, "gradient_updates": 588000, "n_success": 45, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2325, "gain": 0.0005000000000000282, "observed": true, "imputed": false}
80
+ {"eval_index": 79, "rl_env_steps": 1582525, "target_step": 1580000, "sr": 0.24, "round": 198, "update_index": 594000, "policy_version": 198, "chunk_transitions": 198000, "gradient_updates": 594000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23099999999999996, "gain": -0.003500000000000031, "observed": true, "imputed": false}
81
+ {"eval_index": 80, "rl_env_steps": 1606519, "target_step": 1600000, "sr": 0.19, "round": 201, "update_index": 603000, "policy_version": 201, "chunk_transitions": 201000, "gradient_updates": 603000, "n_success": 38, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22849999999999998, "gain": -0.0010000000000000009, "observed": true, "imputed": false}
RL/ToolHang100/QC/mixed100/seed_1/convergence.json ADDED
@@ -0,0 +1,367 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "evals": 81,
3
+ "final_eval": {
4
+ "episodes": 300,
5
+ "imputed": false,
6
+ "n_success": 70,
7
+ "observed": true,
8
+ "seed_start": 800000,
9
+ "sr": 0.23333333333333334,
10
+ "successes": [
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 1,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0,
25
+ 0,
26
+ 0,
27
+ 0,
28
+ 0,
29
+ 0,
30
+ 1,
31
+ 1,
32
+ 0,
33
+ 1,
34
+ 1,
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 1,
40
+ 0,
41
+ 1,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 1,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 0,
59
+ 0,
60
+ 1,
61
+ 0,
62
+ 0,
63
+ 0,
64
+ 0,
65
+ 0,
66
+ 0,
67
+ 1,
68
+ 0,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 0,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 0,
82
+ 1,
83
+ 0,
84
+ 0,
85
+ 1,
86
+ 0,
87
+ 0,
88
+ 0,
89
+ 0,
90
+ 0,
91
+ 1,
92
+ 0,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 0,
98
+ 0,
99
+ 0,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 0,
104
+ 1,
105
+ 0,
106
+ 0,
107
+ 1,
108
+ 0,
109
+ 1,
110
+ 1,
111
+ 0,
112
+ 1,
113
+ 0,
114
+ 0,
115
+ 0,
116
+ 0,
117
+ 1,
118
+ 0,
119
+ 1,
120
+ 0,
121
+ 0,
122
+ 0,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 1,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 0,
132
+ 0,
133
+ 1,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 1,
141
+ 0,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 0,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 0,
155
+ 0,
156
+ 0,
157
+ 0,
158
+ 0,
159
+ 0,
160
+ 0,
161
+ 0,
162
+ 0,
163
+ 1,
164
+ 1,
165
+ 0,
166
+ 0,
167
+ 1,
168
+ 0,
169
+ 0,
170
+ 0,
171
+ 0,
172
+ 0,
173
+ 1,
174
+ 0,
175
+ 0,
176
+ 1,
177
+ 0,
178
+ 1,
179
+ 1,
180
+ 0,
181
+ 0,
182
+ 0,
183
+ 0,
184
+ 0,
185
+ 0,
186
+ 0,
187
+ 0,
188
+ 0,
189
+ 1,
190
+ 0,
191
+ 0,
192
+ 1,
193
+ 1,
194
+ 0,
195
+ 0,
196
+ 0,
197
+ 0,
198
+ 0,
199
+ 0,
200
+ 1,
201
+ 0,
202
+ 1,
203
+ 0,
204
+ 0,
205
+ 0,
206
+ 1,
207
+ 0,
208
+ 0,
209
+ 0,
210
+ 1,
211
+ 0,
212
+ 0,
213
+ 0,
214
+ 0,
215
+ 0,
216
+ 0,
217
+ 0,
218
+ 1,
219
+ 1,
220
+ 0,
221
+ 0,
222
+ 1,
223
+ 1,
224
+ 0,
225
+ 0,
226
+ 0,
227
+ 0,
228
+ 0,
229
+ 0,
230
+ 1,
231
+ 0,
232
+ 0,
233
+ 0,
234
+ 0,
235
+ 1,
236
+ 1,
237
+ 0,
238
+ 1,
239
+ 0,
240
+ 0,
241
+ 0,
242
+ 0,
243
+ 1,
244
+ 0,
245
+ 1,
246
+ 0,
247
+ 0,
248
+ 0,
249
+ 0,
250
+ 1,
251
+ 0,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 0,
256
+ 0,
257
+ 0,
258
+ 0,
259
+ 1,
260
+ 0,
261
+ 0,
262
+ 0,
263
+ 0,
264
+ 0,
265
+ 0,
266
+ 1,
267
+ 0,
268
+ 1,
269
+ 0,
270
+ 0,
271
+ 1,
272
+ 1,
273
+ 1,
274
+ 0,
275
+ 0,
276
+ 0,
277
+ 0,
278
+ 0,
279
+ 0,
280
+ 0,
281
+ 0,
282
+ 0,
283
+ 1,
284
+ 0,
285
+ 0,
286
+ 0,
287
+ 1,
288
+ 0,
289
+ 0,
290
+ 1,
291
+ 1,
292
+ 0,
293
+ 0,
294
+ 0,
295
+ 0,
296
+ 0,
297
+ 0,
298
+ 0,
299
+ 0,
300
+ 0,
301
+ 1,
302
+ 0,
303
+ 0,
304
+ 0,
305
+ 1,
306
+ 0,
307
+ 0,
308
+ 1,
309
+ 0,
310
+ 0
311
+ ]
312
+ },
313
+ "last": {
314
+ "chunk_transitions": 201000,
315
+ "episodes": 200,
316
+ "eval_index": 80,
317
+ "gain": -0.0010000000000000009,
318
+ "gradient_updates": 603000,
319
+ "ma": 0.22849999999999998,
320
+ "n_success": 38,
321
+ "policy_version": 201,
322
+ "rl_env_steps": 1606519,
323
+ "round": 201,
324
+ "seed_start": 960000,
325
+ "sr": 0.19,
326
+ "success_requested": 0,
327
+ "success_requested_imitation": 0,
328
+ "success_rounds_with_skips": 0,
329
+ "success_skipped": 0,
330
+ "success_skipped_imitation": 0,
331
+ "success_skipped_td": 0,
332
+ "target_step": 1600000,
333
+ "update_index": 603000
334
+ },
335
+ "rule": {
336
+ "early_stop": false,
337
+ "every_env_steps": 20000,
338
+ "gain_lag": 10,
339
+ "high_sr": 0.95,
340
+ "high_sr_evals": 5,
341
+ "include_step0": true,
342
+ "ma_window": 10,
343
+ "max_env_steps": 1606196,
344
+ "min_env_steps": 200000,
345
+ "min_gain": 0.02,
346
+ "zero_success_evals": 8
347
+ },
348
+ "status": "budget_complete",
349
+ "stop": {
350
+ "eval_index": 80,
351
+ "gain": -0.0010000000000000009,
352
+ "ma": 0.22849999999999998,
353
+ "rl_env_steps": 1606519,
354
+ "sr": 0.19,
355
+ "status": "budget_complete",
356
+ "trend": "flat"
357
+ },
358
+ "would_stop": {
359
+ "eval_index": 19,
360
+ "gain": -0.0040000000000000036,
361
+ "ma": 0.2525,
362
+ "rl_env_steps": 383661,
363
+ "sr": 0.235,
364
+ "status": "converged_plateau",
365
+ "trend": "flat"
366
+ }
367
+ }
RL/ToolHang100/QC/mixed100/seed_1/convergence.jsonl ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"chunk_transitions": 0, "episodes": 200, "eval_index": 0, "gain": null, "gradient_updates": 0, "ma": null, "n_success": 33, "policy_version": 0, "rl_env_steps": 0, "round": 0, "seed_start": 960000, "sr": 0.165, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 0, "update_index": 0}
2
+ {"chunk_transitions": 3000, "episodes": 200, "eval_index": 1, "gain": null, "gradient_updates": 9000, "ma": null, "n_success": 41, "policy_version": 3, "rl_env_steps": 23974, "round": 3, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 20000, "update_index": 9000}
3
+ {"chunk_transitions": 6000, "episodes": 200, "eval_index": 2, "gain": null, "gradient_updates": 18000, "ma": null, "n_success": 34, "policy_version": 6, "rl_env_steps": 47941, "round": 6, "seed_start": 960000, "sr": 0.17, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 40000, "update_index": 18000}
4
+ {"chunk_transitions": 8000, "episodes": 200, "eval_index": 3, "gain": null, "gradient_updates": 24000, "ma": null, "n_success": 62, "policy_version": 8, "rl_env_steps": 63941, "round": 8, "seed_start": 960000, "sr": 0.31, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 60000, "update_index": 24000}
5
+ {"chunk_transitions": 12000, "episodes": 200, "eval_index": 4, "gain": null, "gradient_updates": 36000, "ma": null, "n_success": 42, "policy_version": 12, "rl_env_steps": 95912, "round": 12, "seed_start": 960000, "sr": 0.21, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 80000, "update_index": 36000}
6
+ {"chunk_transitions": 13000, "episodes": 200, "eval_index": 5, "gain": null, "gradient_updates": 39000, "ma": null, "n_success": 57, "policy_version": 13, "rl_env_steps": 103912, "round": 13, "seed_start": 960000, "sr": 0.285, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 100000, "update_index": 39000}
7
+ {"chunk_transitions": 17000, "episodes": 200, "eval_index": 6, "gain": null, "gradient_updates": 51000, "ma": null, "n_success": 59, "policy_version": 17, "rl_env_steps": 135890, "round": 17, "seed_start": 960000, "sr": 0.295, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 120000, "update_index": 51000}
8
+ {"chunk_transitions": 18000, "episodes": 200, "eval_index": 7, "gain": null, "gradient_updates": 54000, "ma": null, "n_success": 66, "policy_version": 18, "rl_env_steps": 143879, "round": 18, "seed_start": 960000, "sr": 0.33, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 140000, "update_index": 54000}
9
+ {"chunk_transitions": 21000, "episodes": 200, "eval_index": 8, "gain": null, "gradient_updates": 63000, "ma": null, "n_success": 64, "policy_version": 21, "rl_env_steps": 167846, "round": 21, "seed_start": 960000, "sr": 0.32, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 160000, "update_index": 63000}
10
+ {"chunk_transitions": 23000, "episodes": 200, "eval_index": 9, "gain": null, "gradient_updates": 69000, "ma": 0.2565, "n_success": 55, "policy_version": 23, "rl_env_steps": 183840, "round": 23, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 180000, "update_index": 69000}
11
+ {"chunk_transitions": 26000, "episodes": 200, "eval_index": 10, "gain": null, "gradient_updates": 78000, "ma": 0.267, "n_success": 54, "policy_version": 26, "rl_env_steps": 207819, "round": 26, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 200000, "update_index": 78000}
12
+ {"chunk_transitions": 28000, "episodes": 200, "eval_index": 11, "gain": null, "gradient_updates": 84000, "ma": 0.27599999999999997, "n_success": 59, "policy_version": 28, "rl_env_steps": 223806, "round": 28, "seed_start": 960000, "sr": 0.295, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 220000, "update_index": 84000}
13
+ {"chunk_transitions": 31000, "episodes": 200, "eval_index": 12, "gain": null, "gradient_updates": 93000, "ma": 0.289, "n_success": 60, "policy_version": 31, "rl_env_steps": 247784, "round": 31, "seed_start": 960000, "sr": 0.3, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 240000, "update_index": 93000}
14
+ {"chunk_transitions": 33000, "episodes": 200, "eval_index": 13, "gain": null, "gradient_updates": 99000, "ma": 0.28200000000000003, "n_success": 48, "policy_version": 33, "rl_env_steps": 263759, "round": 33, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 260000, "update_index": 99000}
15
+ {"chunk_transitions": 36000, "episodes": 200, "eval_index": 14, "gain": null, "gradient_updates": 108000, "ma": 0.28200000000000003, "n_success": 42, "policy_version": 36, "rl_env_steps": 287728, "round": 36, "seed_start": 960000, "sr": 0.21, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 280000, "update_index": 108000}
16
+ {"chunk_transitions": 38000, "episodes": 200, "eval_index": 15, "gain": null, "gradient_updates": 114000, "ma": 0.279, "n_success": 51, "policy_version": 38, "rl_env_steps": 303728, "round": 38, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 300000, "update_index": 114000}
17
+ {"chunk_transitions": 41000, "episodes": 200, "eval_index": 16, "gain": null, "gradient_updates": 123000, "ma": 0.277, "n_success": 55, "policy_version": 41, "rl_env_steps": 327708, "round": 41, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 320000, "update_index": 123000}
18
+ {"chunk_transitions": 43000, "episodes": 200, "eval_index": 17, "gain": null, "gradient_updates": 129000, "ma": 0.266, "n_success": 44, "policy_version": 43, "rl_env_steps": 343708, "round": 43, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 340000, "update_index": 129000}
19
+ {"chunk_transitions": 46000, "episodes": 200, "eval_index": 18, "gain": null, "gradient_updates": 138000, "ma": 0.25650000000000006, "n_success": 45, "policy_version": 46, "rl_env_steps": 367678, "round": 46, "seed_start": 960000, "sr": 0.225, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 360000, "update_index": 138000}
20
+ {"chunk_transitions": 48000, "episodes": 200, "eval_index": 19, "gain": -0.0040000000000000036, "gradient_updates": 144000, "ma": 0.2525, "n_success": 47, "policy_version": 48, "rl_env_steps": 383661, "round": 48, "seed_start": 960000, "sr": 0.235, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 380000, "update_index": 144000}
21
+ {"chunk_transitions": 51000, "episodes": 200, "eval_index": 20, "gain": -0.015000000000000013, "gradient_updates": 153000, "ma": 0.252, "n_success": 53, "policy_version": 51, "rl_env_steps": 407624, "round": 51, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 400000, "update_index": 153000}
22
+ {"chunk_transitions": 53000, "episodes": 200, "eval_index": 21, "gain": -0.03199999999999997, "gradient_updates": 159000, "ma": 0.244, "n_success": 43, "policy_version": 53, "rl_env_steps": 423609, "round": 53, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 420000, "update_index": 159000}
23
+ {"chunk_transitions": 56000, "episodes": 200, "eval_index": 22, "gain": -0.04949999999999999, "gradient_updates": 168000, "ma": 0.2395, "n_success": 51, "policy_version": 56, "rl_env_steps": 447596, "round": 56, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 440000, "update_index": 168000}
24
+ {"chunk_transitions": 58000, "episodes": 200, "eval_index": 23, "gain": -0.043000000000000066, "gradient_updates": 174000, "ma": 0.23899999999999996, "n_success": 47, "policy_version": 58, "rl_env_steps": 463587, "round": 58, "seed_start": 960000, "sr": 0.235, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 460000, "update_index": 174000}
25
+ {"chunk_transitions": 61000, "episodes": 200, "eval_index": 24, "gain": -0.043999999999999984, "gradient_updates": 183000, "ma": 0.23800000000000004, "n_success": 40, "policy_version": 61, "rl_env_steps": 487565, "round": 61, "seed_start": 960000, "sr": 0.2, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 480000, "update_index": 183000}
26
+ {"chunk_transitions": 63000, "episodes": 200, "eval_index": 25, "gain": -0.04500000000000004, "gradient_updates": 189000, "ma": 0.23399999999999999, "n_success": 43, "policy_version": 63, "rl_env_steps": 503550, "round": 63, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 500000, "update_index": 189000}
27
+ {"chunk_transitions": 66000, "episodes": 200, "eval_index": 26, "gain": -0.04350000000000004, "gradient_updates": 198000, "ma": 0.23349999999999999, "n_success": 54, "policy_version": 66, "rl_env_steps": 527543, "round": 66, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 520000, "update_index": 198000}
28
+ {"chunk_transitions": 68000, "episodes": 200, "eval_index": 27, "gain": -0.03049999999999997, "gradient_updates": 204000, "ma": 0.23550000000000004, "n_success": 48, "policy_version": 68, "rl_env_steps": 543525, "round": 68, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 540000, "update_index": 204000}
29
+ {"chunk_transitions": 71000, "episodes": 200, "eval_index": 28, "gain": -0.011000000000000065, "gradient_updates": 213000, "ma": 0.2455, "n_success": 65, "policy_version": 71, "rl_env_steps": 567508, "round": 71, "seed_start": 960000, "sr": 0.325, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 560000, "update_index": 213000}
30
+ {"chunk_transitions": 73000, "episodes": 200, "eval_index": 29, "gain": -0.005999999999999978, "gradient_updates": 219000, "ma": 0.24650000000000002, "n_success": 49, "policy_version": 73, "rl_env_steps": 583487, "round": 73, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 580000, "update_index": 219000}
31
+ {"chunk_transitions": 76000, "episodes": 200, "eval_index": 30, "gain": -0.004000000000000059, "gradient_updates": 228000, "ma": 0.24799999999999994, "n_success": 56, "policy_version": 76, "rl_env_steps": 607480, "round": 76, "seed_start": 960000, "sr": 0.28, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 600000, "update_index": 228000}
32
+ {"chunk_transitions": 78000, "episodes": 200, "eval_index": 31, "gain": 0.006500000000000006, "gradient_updates": 234000, "ma": 0.2505, "n_success": 48, "policy_version": 78, "rl_env_steps": 623463, "round": 78, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 620000, "update_index": 234000}
33
+ {"chunk_transitions": 81000, "episodes": 200, "eval_index": 32, "gain": 0.01050000000000001, "gradient_updates": 243000, "ma": 0.25, "n_success": 50, "policy_version": 81, "rl_env_steps": 647437, "round": 81, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 640000, "update_index": 243000}
34
+ {"chunk_transitions": 83000, "episodes": 200, "eval_index": 33, "gain": 0.004500000000000032, "gradient_updates": 249000, "ma": 0.2435, "n_success": 34, "policy_version": 83, "rl_env_steps": 663425, "round": 83, "seed_start": 960000, "sr": 0.17, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 660000, "update_index": 249000}
35
+ {"chunk_transitions": 86000, "episodes": 200, "eval_index": 34, "gain": 0.010499999999999954, "gradient_updates": 258000, "ma": 0.2485, "n_success": 50, "policy_version": 86, "rl_env_steps": 687396, "round": 86, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 680000, "update_index": 258000}
36
+ {"chunk_transitions": 88000, "episodes": 200, "eval_index": 35, "gain": 0.015500000000000014, "gradient_updates": 264000, "ma": 0.2495, "n_success": 45, "policy_version": 88, "rl_env_steps": 703373, "round": 88, "seed_start": 960000, "sr": 0.225, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 700000, "update_index": 264000}
37
+ {"chunk_transitions": 91000, "episodes": 200, "eval_index": 36, "gain": 0.015500000000000042, "gradient_updates": 273000, "ma": 0.24900000000000003, "n_success": 53, "policy_version": 91, "rl_env_steps": 727364, "round": 91, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 720000, "update_index": 273000}
38
+ {"chunk_transitions": 93000, "episodes": 200, "eval_index": 37, "gain": 0.010999999999999954, "gradient_updates": 279000, "ma": 0.2465, "n_success": 43, "policy_version": 93, "rl_env_steps": 743345, "round": 93, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 740000, "update_index": 279000}
39
+ {"chunk_transitions": 96000, "episodes": 200, "eval_index": 38, "gain": -0.0040000000000000036, "gradient_updates": 288000, "ma": 0.2415, "n_success": 55, "policy_version": 96, "rl_env_steps": 767331, "round": 96, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 760000, "update_index": 288000}
40
+ {"chunk_transitions": 98000, "episodes": 200, "eval_index": 39, "gain": -0.0010000000000000286, "gradient_updates": 294000, "ma": 0.2455, "n_success": 57, "policy_version": 98, "rl_env_steps": 783313, "round": 98, "seed_start": 960000, "sr": 0.285, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 780000, "update_index": 294000}
41
+ {"chunk_transitions": 101000, "episodes": 200, "eval_index": 40, "gain": -0.005999999999999894, "gradient_updates": 303000, "ma": 0.24200000000000005, "n_success": 49, "policy_version": 101, "rl_env_steps": 807291, "round": 101, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 800000, "update_index": 303000}
42
+ {"chunk_transitions": 103000, "episodes": 200, "eval_index": 41, "gain": -0.007000000000000006, "gradient_updates": 309000, "ma": 0.2435, "n_success": 51, "policy_version": 103, "rl_env_steps": 823274, "round": 103, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 820000, "update_index": 309000}
43
+ {"chunk_transitions": 106000, "episodes": 200, "eval_index": 42, "gain": -0.006000000000000005, "gradient_updates": 318000, "ma": 0.244, "n_success": 51, "policy_version": 106, "rl_env_steps": 847246, "round": 106, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 840000, "update_index": 318000}
44
+ {"chunk_transitions": 108000, "episodes": 200, "eval_index": 43, "gain": 0.0040000000000000036, "gradient_updates": 324000, "ma": 0.2475, "n_success": 41, "policy_version": 108, "rl_env_steps": 863227, "round": 108, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 860000, "update_index": 324000}
45
+ {"chunk_transitions": 111000, "episodes": 200, "eval_index": 44, "gain": -0.0005000000000000004, "gradient_updates": 333000, "ma": 0.248, "n_success": 51, "policy_version": 111, "rl_env_steps": 887199, "round": 111, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 880000, "update_index": 333000}
46
+ {"chunk_transitions": 113000, "episodes": 200, "eval_index": 45, "gain": -0.004500000000000032, "gradient_updates": 339000, "ma": 0.24499999999999997, "n_success": 39, "policy_version": 113, "rl_env_steps": 903199, "round": 113, "seed_start": 960000, "sr": 0.195, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 900000, "update_index": 339000}
47
+ {"chunk_transitions": 116000, "episodes": 200, "eval_index": 46, "gain": -0.011500000000000038, "gradient_updates": 348000, "ma": 0.2375, "n_success": 38, "policy_version": 116, "rl_env_steps": 927186, "round": 116, "seed_start": 960000, "sr": 0.19, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 920000, "update_index": 348000}
48
+ {"chunk_transitions": 118000, "episodes": 200, "eval_index": 47, "gain": -0.0030000000000000027, "gradient_updates": 354000, "ma": 0.2435, "n_success": 55, "policy_version": 118, "rl_env_steps": 943184, "round": 118, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 940000, "update_index": 354000}
49
+ {"chunk_transitions": 121000, "episodes": 200, "eval_index": 48, "gain": -0.0020000000000000018, "gradient_updates": 363000, "ma": 0.2395, "n_success": 47, "policy_version": 121, "rl_env_steps": 967152, "round": 121, "seed_start": 960000, "sr": 0.235, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 960000, "update_index": 363000}
50
+ {"chunk_transitions": 123000, "episodes": 200, "eval_index": 49, "gain": -0.007000000000000006, "gradient_updates": 369000, "ma": 0.2385, "n_success": 55, "policy_version": 123, "rl_env_steps": 983141, "round": 123, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 980000, "update_index": 369000}
51
+ {"chunk_transitions": 126000, "episodes": 200, "eval_index": 50, "gain": -0.0020000000000000018, "gradient_updates": 378000, "ma": 0.24000000000000005, "n_success": 52, "policy_version": 126, "rl_env_steps": 1007110, "round": 126, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1000000, "update_index": 378000}
52
+ {"chunk_transitions": 128000, "episodes": 200, "eval_index": 51, "gain": -0.002999999999999975, "gradient_updates": 384000, "ma": 0.24050000000000002, "n_success": 52, "policy_version": 128, "rl_env_steps": 1023084, "round": 128, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1020000, "update_index": 384000}
53
+ {"chunk_transitions": 131000, "episodes": 200, "eval_index": 52, "gain": -0.005000000000000032, "gradient_updates": 393000, "ma": 0.23899999999999996, "n_success": 48, "policy_version": 131, "rl_env_steps": 1047060, "round": 131, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1040000, "update_index": 393000}
54
+ {"chunk_transitions": 133000, "episodes": 200, "eval_index": 53, "gain": -0.0040000000000000036, "gradient_updates": 399000, "ma": 0.2435, "n_success": 50, "policy_version": 133, "rl_env_steps": 1063023, "round": 133, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1060000, "update_index": 399000}
55
+ {"chunk_transitions": 136000, "episodes": 200, "eval_index": 54, "gain": -0.003999999999999948, "gradient_updates": 408000, "ma": 0.24400000000000005, "n_success": 52, "policy_version": 136, "rl_env_steps": 1087011, "round": 136, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1080000, "update_index": 408000}
56
+ {"chunk_transitions": 138000, "episodes": 200, "eval_index": 55, "gain": 0.008500000000000035, "gradient_updates": 414000, "ma": 0.2535, "n_success": 58, "policy_version": 138, "rl_env_steps": 1102994, "round": 138, "seed_start": 960000, "sr": 0.29, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1100000, "update_index": 414000}
57
+ {"chunk_transitions": 141000, "episodes": 200, "eval_index": 56, "gain": 0.02100000000000002, "gradient_updates": 423000, "ma": 0.2585, "n_success": 48, "policy_version": 141, "rl_env_steps": 1126981, "round": 141, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1120000, "update_index": 423000}
58
+ {"chunk_transitions": 143000, "episodes": 200, "eval_index": 57, "gain": 0.010000000000000064, "gradient_updates": 429000, "ma": 0.25350000000000006, "n_success": 45, "policy_version": 143, "rl_env_steps": 1142956, "round": 143, "seed_start": 960000, "sr": 0.225, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1140000, "update_index": 429000}
59
+ {"chunk_transitions": 147000, "episodes": 200, "eval_index": 58, "gain": 0.015000000000000069, "gradient_updates": 441000, "ma": 0.25450000000000006, "n_success": 49, "policy_version": 147, "rl_env_steps": 1174936, "round": 147, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1160000, "update_index": 441000}
60
+ {"chunk_transitions": 148000, "episodes": 200, "eval_index": 59, "gain": 0.011500000000000066, "gradient_updates": 444000, "ma": 0.25000000000000006, "n_success": 46, "policy_version": 148, "rl_env_steps": 1182936, "round": 148, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1180000, "update_index": 444000}
61
+ {"chunk_transitions": 151000, "episodes": 200, "eval_index": 60, "gain": 0.010499999999999954, "gradient_updates": 453000, "ma": 0.2505, "n_success": 53, "policy_version": 151, "rl_env_steps": 1206913, "round": 151, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1200000, "update_index": 453000}
62
+ {"chunk_transitions": 153000, "episodes": 200, "eval_index": 61, "gain": 0.002999999999999975, "gradient_updates": 459000, "ma": 0.2435, "n_success": 38, "policy_version": 153, "rl_env_steps": 1222893, "round": 153, "seed_start": 960000, "sr": 0.19, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1220000, "update_index": 459000}
63
+ {"chunk_transitions": 156000, "episodes": 200, "eval_index": 62, "gain": 0.00250000000000003, "gradient_updates": 468000, "ma": 0.2415, "n_success": 44, "policy_version": 156, "rl_env_steps": 1246875, "round": 156, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1240000, "update_index": 468000}
64
+ {"chunk_transitions": 158000, "episodes": 200, "eval_index": 63, "gain": -0.006000000000000005, "gradient_updates": 474000, "ma": 0.2375, "n_success": 42, "policy_version": 158, "rl_env_steps": 1262860, "round": 158, "seed_start": 960000, "sr": 0.21, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1260000, "update_index": 474000}
65
+ {"chunk_transitions": 161000, "episodes": 200, "eval_index": 64, "gain": -0.006500000000000061, "gradient_updates": 483000, "ma": 0.2375, "n_success": 52, "policy_version": 161, "rl_env_steps": 1286833, "round": 161, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1280000, "update_index": 483000}
66
+ {"chunk_transitions": 163000, "episodes": 200, "eval_index": 65, "gain": -0.016499999999999987, "gradient_updates": 489000, "ma": 0.23700000000000002, "n_success": 57, "policy_version": 163, "rl_env_steps": 1302817, "round": 163, "seed_start": 960000, "sr": 0.285, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1300000, "update_index": 489000}
67
+ {"chunk_transitions": 166000, "episodes": 200, "eval_index": 66, "gain": -0.025499999999999995, "gradient_updates": 498000, "ma": 0.233, "n_success": 40, "policy_version": 166, "rl_env_steps": 1326788, "round": 166, "seed_start": 960000, "sr": 0.2, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1320000, "update_index": 498000}
68
+ {"chunk_transitions": 168000, "episodes": 200, "eval_index": 67, "gain": -0.02450000000000005, "gradient_updates": 504000, "ma": 0.229, "n_success": 37, "policy_version": 168, "rl_env_steps": 1342777, "round": 168, "seed_start": 960000, "sr": 0.185, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1340000, "update_index": 504000}
69
+ {"chunk_transitions": 171000, "episodes": 200, "eval_index": 68, "gain": -0.022500000000000075, "gradient_updates": 513000, "ma": 0.23199999999999998, "n_success": 55, "policy_version": 171, "rl_env_steps": 1366763, "round": 171, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1360000, "update_index": 513000}
70
+ {"chunk_transitions": 173000, "episodes": 200, "eval_index": 69, "gain": -0.01550000000000007, "gradient_updates": 519000, "ma": 0.2345, "n_success": 51, "policy_version": 173, "rl_env_steps": 1382739, "round": 173, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1380000, "update_index": 519000}
71
+ {"chunk_transitions": 176000, "episodes": 200, "eval_index": 70, "gain": -0.02100000000000002, "gradient_updates": 528000, "ma": 0.22949999999999998, "n_success": 43, "policy_version": 176, "rl_env_steps": 1406718, "round": 176, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1400000, "update_index": 528000}
72
+ {"chunk_transitions": 178000, "episodes": 200, "eval_index": 71, "gain": -0.012499999999999983, "gradient_updates": 534000, "ma": 0.231, "n_success": 41, "policy_version": 178, "rl_env_steps": 1422707, "round": 178, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1420000, "update_index": 534000}
73
+ {"chunk_transitions": 181000, "episodes": 200, "eval_index": 72, "gain": -0.006000000000000005, "gradient_updates": 543000, "ma": 0.2355, "n_success": 53, "policy_version": 181, "rl_env_steps": 1446668, "round": 181, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1440000, "update_index": 543000}
74
+ {"chunk_transitions": 183000, "episodes": 200, "eval_index": 73, "gain": -0.008000000000000007, "gradient_updates": 549000, "ma": 0.22949999999999998, "n_success": 30, "policy_version": 183, "rl_env_steps": 1462642, "round": 183, "seed_start": 960000, "sr": 0.15, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1460000, "update_index": 549000}
75
+ {"chunk_transitions": 186000, "episodes": 200, "eval_index": 74, "gain": -0.006499999999999978, "gradient_updates": 558000, "ma": 0.231, "n_success": 55, "policy_version": 186, "rl_env_steps": 1486630, "round": 186, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1480000, "update_index": 558000}
76
+ {"chunk_transitions": 188000, "episodes": 200, "eval_index": 75, "gain": -0.007500000000000034, "gradient_updates": 564000, "ma": 0.22949999999999998, "n_success": 54, "policy_version": 188, "rl_env_steps": 1502611, "round": 188, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1500000, "update_index": 564000}
77
+ {"chunk_transitions": 191000, "episodes": 200, "eval_index": 76, "gain": -5.551115123125783e-17, "gradient_updates": 573000, "ma": 0.23299999999999996, "n_success": 47, "policy_version": 191, "rl_env_steps": 1526589, "round": 191, "seed_start": 960000, "sr": 0.235, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1520000, "update_index": 573000}
78
+ {"chunk_transitions": 193000, "episodes": 200, "eval_index": 77, "gain": 0.00849999999999998, "gradient_updates": 579000, "ma": 0.2375, "n_success": 46, "policy_version": 193, "rl_env_steps": 1542576, "round": 193, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1540000, "update_index": 579000}
79
+ {"chunk_transitions": 196000, "episodes": 200, "eval_index": 78, "gain": 0.0005000000000000282, "gradient_updates": 588000, "ma": 0.2325, "n_success": 45, "policy_version": 196, "rl_env_steps": 1566542, "round": 196, "seed_start": 960000, "sr": 0.225, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1560000, "update_index": 588000}
80
+ {"chunk_transitions": 198000, "episodes": 200, "eval_index": 79, "gain": -0.003500000000000031, "gradient_updates": 594000, "ma": 0.23099999999999996, "n_success": 48, "policy_version": 198, "rl_env_steps": 1582525, "round": 198, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1580000, "update_index": 594000}
81
+ {"chunk_transitions": 201000, "episodes": 200, "eval_index": 80, "gain": -0.0010000000000000009, "gradient_updates": 603000, "ma": 0.22849999999999998, "n_success": 38, "policy_version": 201, "rl_env_steps": 1606519, "round": 201, "seed_start": 960000, "sr": 0.19, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1600000, "update_index": 603000}
RL/ToolHang100/QC/mixed100/seed_1/eval/final.parquet ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c11af8b26b66ce4ebe4d0a7f7ae3fc93aaef13fd8ac5dbf5920b8318fb2c923a
3
+ size 6768
RL/ToolHang100/QC/mixed100/seed_1/final/source_metadata.json ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_sha256": "70dda4b58013596119a7b3bbb3d85a2f871fba438451a99f2754c18290928ead",
3
+ "evaluation": {
4
+ "episodes": 300,
5
+ "imputed": false,
6
+ "n_success": 70,
7
+ "observed": true,
8
+ "seed_start": 800000,
9
+ "sr": 0.23333333333333334,
10
+ "successes": [
11
+ 0,
12
+ 0,
13
+ 0,
14
+ 0,
15
+ 0,
16
+ 0,
17
+ 0,
18
+ 0,
19
+ 1,
20
+ 0,
21
+ 0,
22
+ 0,
23
+ 0,
24
+ 0,
25
+ 0,
26
+ 0,
27
+ 0,
28
+ 0,
29
+ 0,
30
+ 1,
31
+ 1,
32
+ 0,
33
+ 1,
34
+ 1,
35
+ 0,
36
+ 0,
37
+ 0,
38
+ 0,
39
+ 1,
40
+ 0,
41
+ 1,
42
+ 0,
43
+ 0,
44
+ 0,
45
+ 0,
46
+ 0,
47
+ 0,
48
+ 0,
49
+ 1,
50
+ 0,
51
+ 0,
52
+ 0,
53
+ 0,
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 0,
58
+ 0,
59
+ 0,
60
+ 1,
61
+ 0,
62
+ 0,
63
+ 0,
64
+ 0,
65
+ 0,
66
+ 0,
67
+ 1,
68
+ 0,
69
+ 0,
70
+ 0,
71
+ 0,
72
+ 0,
73
+ 0,
74
+ 0,
75
+ 0,
76
+ 0,
77
+ 0,
78
+ 0,
79
+ 0,
80
+ 0,
81
+ 0,
82
+ 1,
83
+ 0,
84
+ 0,
85
+ 1,
86
+ 0,
87
+ 0,
88
+ 0,
89
+ 0,
90
+ 0,
91
+ 1,
92
+ 0,
93
+ 0,
94
+ 0,
95
+ 0,
96
+ 0,
97
+ 0,
98
+ 0,
99
+ 0,
100
+ 1,
101
+ 1,
102
+ 1,
103
+ 0,
104
+ 1,
105
+ 0,
106
+ 0,
107
+ 1,
108
+ 0,
109
+ 1,
110
+ 1,
111
+ 0,
112
+ 1,
113
+ 0,
114
+ 0,
115
+ 0,
116
+ 0,
117
+ 1,
118
+ 0,
119
+ 1,
120
+ 0,
121
+ 0,
122
+ 0,
123
+ 0,
124
+ 0,
125
+ 0,
126
+ 0,
127
+ 1,
128
+ 0,
129
+ 0,
130
+ 0,
131
+ 0,
132
+ 0,
133
+ 1,
134
+ 0,
135
+ 0,
136
+ 0,
137
+ 0,
138
+ 0,
139
+ 0,
140
+ 1,
141
+ 0,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 0,
146
+ 0,
147
+ 0,
148
+ 0,
149
+ 0,
150
+ 0,
151
+ 0,
152
+ 0,
153
+ 0,
154
+ 0,
155
+ 0,
156
+ 0,
157
+ 0,
158
+ 0,
159
+ 0,
160
+ 0,
161
+ 0,
162
+ 0,
163
+ 1,
164
+ 1,
165
+ 0,
166
+ 0,
167
+ 1,
168
+ 0,
169
+ 0,
170
+ 0,
171
+ 0,
172
+ 0,
173
+ 1,
174
+ 0,
175
+ 0,
176
+ 1,
177
+ 0,
178
+ 1,
179
+ 1,
180
+ 0,
181
+ 0,
182
+ 0,
183
+ 0,
184
+ 0,
185
+ 0,
186
+ 0,
187
+ 0,
188
+ 0,
189
+ 1,
190
+ 0,
191
+ 0,
192
+ 1,
193
+ 1,
194
+ 0,
195
+ 0,
196
+ 0,
197
+ 0,
198
+ 0,
199
+ 0,
200
+ 1,
201
+ 0,
202
+ 1,
203
+ 0,
204
+ 0,
205
+ 0,
206
+ 1,
207
+ 0,
208
+ 0,
209
+ 0,
210
+ 1,
211
+ 0,
212
+ 0,
213
+ 0,
214
+ 0,
215
+ 0,
216
+ 0,
217
+ 0,
218
+ 1,
219
+ 1,
220
+ 0,
221
+ 0,
222
+ 1,
223
+ 1,
224
+ 0,
225
+ 0,
226
+ 0,
227
+ 0,
228
+ 0,
229
+ 0,
230
+ 1,
231
+ 0,
232
+ 0,
233
+ 0,
234
+ 0,
235
+ 1,
236
+ 1,
237
+ 0,
238
+ 1,
239
+ 0,
240
+ 0,
241
+ 0,
242
+ 0,
243
+ 1,
244
+ 0,
245
+ 1,
246
+ 0,
247
+ 0,
248
+ 0,
249
+ 0,
250
+ 1,
251
+ 0,
252
+ 1,
253
+ 1,
254
+ 1,
255
+ 0,
256
+ 0,
257
+ 0,
258
+ 0,
259
+ 1,
260
+ 0,
261
+ 0,
262
+ 0,
263
+ 0,
264
+ 0,
265
+ 0,
266
+ 1,
267
+ 0,
268
+ 1,
269
+ 0,
270
+ 0,
271
+ 1,
272
+ 1,
273
+ 1,
274
+ 0,
275
+ 0,
276
+ 0,
277
+ 0,
278
+ 0,
279
+ 0,
280
+ 0,
281
+ 0,
282
+ 0,
283
+ 1,
284
+ 0,
285
+ 0,
286
+ 0,
287
+ 1,
288
+ 0,
289
+ 0,
290
+ 1,
291
+ 1,
292
+ 0,
293
+ 0,
294
+ 0,
295
+ 0,
296
+ 0,
297
+ 0,
298
+ 0,
299
+ 0,
300
+ 0,
301
+ 1,
302
+ 0,
303
+ 0,
304
+ 0,
305
+ 1,
306
+ 0,
307
+ 0,
308
+ 1,
309
+ 0,
310
+ 0
311
+ ]
312
+ },
313
+ "status": "budget_complete"
314
+ }
RL/ToolHang100/QC/mixed100/seed_1/final/source_resolved_config.json ADDED
@@ -0,0 +1,1195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
14
+ "key": "n_step_chunks",
15
+ "reason": "task-specific unified Study2 setting",
16
+ "upstream": 3,
17
+ "value": 5
18
+ },
19
+ {
20
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
21
+ "key": "offline.cohort_size",
22
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
23
+ "upstream": 50,
24
+ "value": 100
25
+ },
26
+ {
27
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
28
+ "key": "offline.episodes",
29
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
30
+ "upstream": 50,
31
+ "value": 100
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.act_bounds_json",
36
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
37
+ "upstream": null,
38
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}"
39
+ },
40
+ {
41
+ "decision_ref": "[0930-S2DEC]",
42
+ "key": "learner.qc.agent.act_max",
43
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
44
+ "upstream": 1.0,
45
+ "value": 3.0794031620025635
46
+ },
47
+ {
48
+ "decision_ref": "[0930-S2DEC]",
49
+ "key": "learner.qc.agent.act_min",
50
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
51
+ "upstream": -1.0,
52
+ "value": -3.0920298099517822
53
+ },
54
+ {
55
+ "decision_ref": "[0930-S2DEC]",
56
+ "key": "learner.qc.agent.actor_lr_partition",
57
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
58
+ "upstream": "lr=0.0003 shared by actor and critic",
59
+ "value": 0.0001
60
+ },
61
+ {
62
+ "decision_ref": "[1002-S2ALIGN]",
63
+ "key": "learner.qc.agent.actor_num_samples",
64
+ "reason": "single BoN knob including QC TD",
65
+ "upstream": 32,
66
+ "value": 16
67
+ },
68
+ {
69
+ "decision_ref": "[1002-S2ALIGN]",
70
+ "key": "learner.qc.agent.best_of_n",
71
+ "reason": "single BoN knob: collection/eval/QC TD",
72
+ "upstream": 8,
73
+ "value": 16
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.bon_agg",
78
+ "reason": "Best-of-N selection by the ensemble mean",
79
+ "upstream": null,
80
+ "value": "mean"
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.bon_sft",
85
+ "reason": "variant qc flag bon_sft",
86
+ "upstream": "upstream_yaml",
87
+ "value": false
88
+ },
89
+ {
90
+ "decision_ref": "[0930-S2DEC]",
91
+ "key": "learner.qc.agent.critic_lr_partition",
92
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
93
+ "upstream": "lr=0.0003 shared by actor and critic",
94
+ "value": 0.0003
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.partition_optimizer",
99
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
100
+ "upstream": null,
101
+ "value": true
102
+ },
103
+ {
104
+ "decision_ref": "docs/scope.md [0928-I]",
105
+ "key": "learner.qc.agent.q_target_agg",
106
+ "reason": "REDQ-style TD target: min over a random critic subset",
107
+ "upstream": null,
108
+ "value": "subset_min"
109
+ },
110
+ {
111
+ "decision_ref": "docs/scope.md [0928-I]",
112
+ "key": "learner.qc.agent.q_target_replace",
113
+ "reason": "subset drawn without replacement",
114
+ "upstream": null,
115
+ "value": false
116
+ },
117
+ {
118
+ "decision_ref": "docs/scope.md [0928-I]",
119
+ "key": "learner.qc.agent.q_target_subset",
120
+ "reason": "TD-target subset size",
121
+ "upstream": null,
122
+ "value": 2
123
+ },
124
+ {
125
+ "decision_ref": "[0930-S2DEC]",
126
+ "key": "learner.qc.agent.tau",
127
+ "reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
128
+ "upstream": 0.05,
129
+ "value": 0.005
130
+ },
131
+ {
132
+ "decision_ref": "[0930-S2DEC]",
133
+ "key": "learner.qc.agent.value_hidden_dims",
134
+ "reason": "unified.critic_hidden (critics only; actors native)",
135
+ "upstream": [
136
+ 512,
137
+ 512,
138
+ 512,
139
+ 512,
140
+ 512
141
+ ],
142
+ "value": [
143
+ 512,
144
+ 512,
145
+ 512
146
+ ]
147
+ },
148
+ {
149
+ "decision_ref": "docs/scope.md [0928-I]",
150
+ "key": "learner.qc.training.bc_pi_steps",
151
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
152
+ "upstream": 1000000,
153
+ "value": 0
154
+ },
155
+ {
156
+ "decision_ref": "docs/scope.md [0928-I]",
157
+ "key": "learner.qc.loop.buffer.utd_online",
158
+ "reason": "UTD=3 per executed action chunk",
159
+ "upstream": 1,
160
+ "value": "3 gradient updates per executed chunk"
161
+ },
162
+ {
163
+ "decision_ref": "docs/scope.md [0928-I]",
164
+ "key": "learner.qc.loop.n_step",
165
+ "reason": "n-step=3 counted in action chunks",
166
+ "upstream": "1 chunk (sequence_length = horizon_length)",
167
+ "value": "5 executed chunks (unified_env_step window)"
168
+ },
169
+ {
170
+ "decision_ref": "docs/scope.md [0928-I]",
171
+ "key": "learner.qc.loop.critic_actor_ratio",
172
+ "reason": "critic every update, actor every 2nd global update",
173
+ "upstream": "1:1 (utd_q = utd_pi)",
174
+ "value": "2:1"
175
+ },
176
+ {
177
+ "decision_ref": "docs/scope.md [0928-I]",
178
+ "key": "learner.qc.loop.buffer.start_training",
179
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
180
+ "upstream": 10000,
181
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
182
+ },
183
+ {
184
+ "decision_ref": "docs/scope.md [0928-I]",
185
+ "key": "learner.qc.loop.buffer.offline_ratio",
186
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
187
+ "upstream": 0.0,
188
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
189
+ },
190
+ {
191
+ "decision_ref": "docs/scope.md [0928-I]",
192
+ "key": "learner.qc.loop.buffer.size",
193
+ "reason": "one shared replay of executed chunks; success buffer = mask",
194
+ "upstream": 2000000,
195
+ "value": "1000000 executed chunks, shared FIFO"
196
+ },
197
+ {
198
+ "decision_ref": "docs/scope.md [0928-I]",
199
+ "key": "learner.qc.actor_architecture",
200
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
201
+ "upstream": [
202
+ 512,
203
+ 512,
204
+ 512,
205
+ 512,
206
+ 512
207
+ ],
208
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
209
+ },
210
+ {
211
+ "decision_ref": "docs/scope.md [0928-I]",
212
+ "key": "learner.qc.observation",
213
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
214
+ "upstream": "low-dim state",
215
+ "value": "frozen Ri features c [137] from the checkpoint encoder"
216
+ },
217
+ {
218
+ "decision_ref": "[0930-S2DEC]",
219
+ "key": "learner.qc.reward",
220
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
221
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
222
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
223
+ },
224
+ {
225
+ "decision_ref": "[0930-S2DEC]",
226
+ "key": "learner.qc.td_target",
227
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
228
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
229
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
230
+ },
231
+ {
232
+ "decision_ref": "[0930-S2DEC]",
233
+ "key": "learner.qc.eval",
234
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
235
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
236
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
237
+ },
238
+ {
239
+ "decision_ref": "[1002-S2ALIGN]",
240
+ "key": "learner.qc.lr_schedule",
241
+ "reason": "RL env-step clock shared with extra success TD",
242
+ "upstream": "native per-optimizer schedule",
243
+ "value": {
244
+ "actor": 0.0001,
245
+ "critic": 0.0003,
246
+ "decay_end_env_steps": 1606196,
247
+ "min_ratio": 0.1,
248
+ "schedule": "warmup_cosine",
249
+ "temperature": 0.0003,
250
+ "warmup_env_steps": 20000
251
+ }
252
+ },
253
+ {
254
+ "decision_ref": "[0930-S2DEC]",
255
+ "key": "learner.qc.agent.q_min_q_max",
256
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
257
+ "upstream": [
258
+ -999.9999999999991,
259
+ 0.0
260
+ ],
261
+ "value": [
262
+ 0.0,
263
+ 1.0
264
+ ]
265
+ }
266
+ ],
267
+ "protocol": {
268
+ "checkpoint_every_rounds": 5,
269
+ "checkpoint_retention": {
270
+ "best": true,
271
+ "collapse_drop": {
272
+ "min_abs": 0.2,
273
+ "ratio": 0.5
274
+ },
275
+ "diagnostics": true,
276
+ "every_evals": 3
277
+ },
278
+ "collection": {
279
+ "n_envs": 4,
280
+ "train_seed_base": 100000000,
281
+ "train_seed_stride": 1000000,
282
+ "training_freq_chunks": 1000
283
+ },
284
+ "convergence": {
285
+ "early_stop": false,
286
+ "gain_lag": 10,
287
+ "high_sr": 0.95,
288
+ "high_sr_evals": 5,
289
+ "ma_window": 10,
290
+ "max_env_steps": 1606196,
291
+ "min_env_steps": 200000,
292
+ "min_gain": 0.02,
293
+ "zero_success_evals": 8
294
+ },
295
+ "critic_ensemble": {
296
+ "num": 10,
297
+ "policy": "mean",
298
+ "target": {
299
+ "kind": "subset_min",
300
+ "replace": false,
301
+ "size": 2
302
+ }
303
+ },
304
+ "eval": {
305
+ "best_of_n": 16,
306
+ "episodes": 200,
307
+ "every_env_steps": 20000,
308
+ "include_step0": true,
309
+ "seed_start": 960000,
310
+ "workers": 4
311
+ },
312
+ "final_eval": {
313
+ "episodes": 300,
314
+ "on_zero_stop": true,
315
+ "seed_start": 800000
316
+ },
317
+ "n_step_chunks": 5,
318
+ "offline": {
319
+ "adaptive_ratio": false,
320
+ "cohort_size": 100,
321
+ "episodes": 100,
322
+ "learners_without_offline": [
323
+ "dsrl_sac"
324
+ ],
325
+ "ratio": {
326
+ "decay_env_steps": {
327
+ "can": 160000,
328
+ "square": 320000,
329
+ "tool_hang": 640000,
330
+ "transport": 320000
331
+ },
332
+ "end": 0.1,
333
+ "start": {
334
+ "can": 0.7,
335
+ "square": 0.5,
336
+ "tool_hang": 0.5,
337
+ "transport": 0.9
338
+ }
339
+ },
340
+ "selection": "ascending_demo_index",
341
+ "source": "bc_regime",
342
+ "unsupported_learners": [
343
+ "zprl"
344
+ ]
345
+ },
346
+ "replay": {
347
+ "capacity_chunks": 1000000,
348
+ "pin_start_successes": false,
349
+ "success_buffer": "success_mask_on_shared_fifo",
350
+ "success_skip_rule": "skip_below_success_batch_size",
351
+ "unit": "executed_chunk"
352
+ },
353
+ "start": {
354
+ "max_episodes": 200,
355
+ "mode": "offline_buffer",
356
+ "policy": {
357
+ "default": "base_cfm_ode",
358
+ "zprl": "vib_base"
359
+ },
360
+ "target_successes": 20
361
+ },
362
+ "unified": {
363
+ "actor_target_tau": 0.005,
364
+ "critic_hidden": [
365
+ 512,
366
+ 512,
367
+ 512
368
+ ],
369
+ "critic_target_every_critic_step": true,
370
+ "critic_target_tau": 0.005,
371
+ "dsrl_noise_imitation_reference_dims": 28,
372
+ "gamma_env": 0.999,
373
+ "lr": {
374
+ "actor": 0.0001,
375
+ "critic": 0.0003,
376
+ "decay_end_env_steps": 1606196,
377
+ "min_ratio": 0.1,
378
+ "schedule": "warmup_cosine",
379
+ "temperature": 0.0003,
380
+ "warmup_env_steps": 20000
381
+ },
382
+ "reward": "binary_first_success",
383
+ "success_td_polyak": true,
384
+ "timeout_bootstrap": false,
385
+ "zprl_imitation": "mse_pretanh_mean"
386
+ },
387
+ "updates": {
388
+ "batch_size": 256,
389
+ "critic_per_actor": 2,
390
+ "success_batch_size": 256,
391
+ "utd_per_chunk": 3
392
+ },
393
+ "version": "study2_protocol_v10"
394
+ },
395
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
396
+ "provenance_extra": {
397
+ "agent_library": {
398
+ "framework": "jax",
399
+ "sources": {
400
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
401
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
402
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
403
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
404
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
405
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
406
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
407
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
408
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
409
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
410
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
411
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
412
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
413
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
414
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
415
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
416
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
417
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
418
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
419
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
420
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
421
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
422
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
423
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
424
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
425
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
426
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
427
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
428
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
429
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
430
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
431
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
432
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
433
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
434
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
435
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
436
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
437
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
438
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
439
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
440
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
441
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
442
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
443
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
444
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
445
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
446
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
447
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
448
+ "train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
449
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
450
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
451
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
452
+ },
453
+ "version": "study2_jax_agents_v2"
454
+ },
455
+ "base_framework": "jax",
456
+ "bc_identity": {
457
+ "architecture": "dice_vision_resnet_flow_unet_v1",
458
+ "family": "StandardBaseBC",
459
+ "framework": "jax"
460
+ },
461
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/base/mixed100/epoch_300.pkl",
462
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
463
+ "experiment": "tool_hang",
464
+ "frozen_hash": "848c66c08785dca4f652c3a7eb0bd2cb26761fbdf91fb1dbcc2cde5be0446931",
465
+ "learner": {
466
+ "action_bounds": {
467
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
468
+ "compat_act_max": 3.0794031620025635,
469
+ "compat_act_min": -3.0920298099517822,
470
+ "hi": [
471
+ 1.0,
472
+ 1.0,
473
+ 1.0,
474
+ 3.0794031620025635,
475
+ 1.9159542322158813,
476
+ 1.0,
477
+ 1.0
478
+ ],
479
+ "kind": "per_dimension",
480
+ "lo": [
481
+ -1.0,
482
+ -1.0,
483
+ -1.0,
484
+ -3.0920298099517822,
485
+ -2.0611963272094727,
486
+ -1.531522274017334,
487
+ -1.0
488
+ ],
489
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
490
+ },
491
+ "agent_config": {
492
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
493
+ "act_max": 3.0794031620025635,
494
+ "act_min": -3.0920298099517822,
495
+ "action_chunking": true,
496
+ "action_critic_hidden_dims": null,
497
+ "action_dim": 7,
498
+ "action_num_qs": 10,
499
+ "action_repeat": false,
500
+ "actor_backbone": "mlp",
501
+ "actor_decay_steps": 50000,
502
+ "actor_end_value": 2e-05,
503
+ "actor_hidden_dims": [
504
+ 512,
505
+ 512,
506
+ 512,
507
+ 512,
508
+ 512
509
+ ],
510
+ "actor_layer_norm": false,
511
+ "actor_lr": 0.0003,
512
+ "actor_lr_partition": 0.0001,
513
+ "actor_num_samples": 16,
514
+ "actor_obs": "state",
515
+ "actor_scheduler": "cosine",
516
+ "actor_tau": 0.005,
517
+ "actor_type": "best-of-n",
518
+ "actor_warmup_steps": 2000,
519
+ "actor_weight_decay": 0.0,
520
+ "adv_clip_min": null,
521
+ "agent_name": "acfql",
522
+ "alpha": 500,
523
+ "batch_size": 256,
524
+ "best_of_n": 16,
525
+ "bon_agg": "mean",
526
+ "bon_sft": false,
527
+ "bon_sft_coeff": 1.0,
528
+ "bootstrap_ratio": 0.5,
529
+ "bptt_num_samples": 4,
530
+ "calql_q_subsample": 2,
531
+ "clip_grad_norm": 1000.0,
532
+ "clip_intermediate_actions": true,
533
+ "clip_max_epsilon_multiplier": 1.0,
534
+ "clip_min_epsilon_multiplier": 1.0,
535
+ "constant_noise_std": 0.01,
536
+ "cql_alpha": 5.0,
537
+ "cql_n_actions": 10,
538
+ "cql_temp": 1.0,
539
+ "critic_backbone": "mlp",
540
+ "critic_decay_steps": 5000,
541
+ "critic_end_value": 1e-08,
542
+ "critic_loss_type": "mse",
543
+ "critic_lr": 0.0003,
544
+ "critic_lr_partition": 0.0003,
545
+ "critic_obs": "state",
546
+ "critic_scheduler": "constant",
547
+ "critic_warmup_steps": 500,
548
+ "critic_weight_decay": 1e-05,
549
+ "denoised_clip_value": 1.0,
550
+ "discount": 0.999,
551
+ "edit_action_scale": 0.5,
552
+ "edit_actor_hidden_dims": null,
553
+ "edit_hidden_dims": [
554
+ 256,
555
+ 256,
556
+ 256
557
+ ],
558
+ "edit_init_temperature": 1.0,
559
+ "edit_layer_norm": true,
560
+ "edit_lr": 0.0003,
561
+ "edit_policy_bound": 0.1,
562
+ "edit_start_step": 100000,
563
+ "edit_target_entropy": null,
564
+ "edit_weight_decay": 0.0,
565
+ "encoder": null,
566
+ "entropy_backup": false,
567
+ "flow_steps": 10,
568
+ "fql_distillation_coeff": 10.0,
569
+ "fql_train_main_policy": true,
570
+ "freeze_vision_encoder": false,
571
+ "ft_flow_steps": 10,
572
+ "horizon_length": 8,
573
+ "init_temperature": 1.0,
574
+ "layer_norm": true,
575
+ "lr": 0.0003,
576
+ "lr_action_critic": null,
577
+ "lr_actor": null,
578
+ "lr_actor_bc_flow": null,
579
+ "lr_critic": null,
580
+ "lr_edit_actor": null,
581
+ "lr_noise_critic": null,
582
+ "max_noise_std": 0.01,
583
+ "mc_regression": false,
584
+ "mc_regression_coeff": 1.0,
585
+ "min_noise_std": 0.01,
586
+ "mip_q_ensemble": false,
587
+ "mip_q_noise_scale": 1.0,
588
+ "mip_q_t_star": 0.9,
589
+ "mip_t_star": 0.999,
590
+ "muon_beta": 0.95,
591
+ "muon_nesterov": true,
592
+ "muon_ns_steps": 5,
593
+ "n_action_samples": 8,
594
+ "n_edit_samples": 8,
595
+ "noise_critic_grad_steps": 1,
596
+ "noise_dim": null,
597
+ "noise_net_hidden_dims": [
598
+ 256,
599
+ 256
600
+ ],
601
+ "noise_net_layer_norm": true,
602
+ "normalize_act_space_dimension": false,
603
+ "normalize_denoising_horizon": false,
604
+ "normalize_q_loss": false,
605
+ "num_bins": 256,
606
+ "num_ensemble_members": 10,
607
+ "num_qs": 10,
608
+ "ob_dims": [
609
+ 137
610
+ ],
611
+ "one_step_decay_steps": null,
612
+ "one_step_end_value": null,
613
+ "one_step_hidden_dims": [
614
+ 512,
615
+ 512,
616
+ 512,
617
+ 512
618
+ ],
619
+ "one_step_layer_norm": false,
620
+ "one_step_lr": null,
621
+ "one_step_scheduler": null,
622
+ "one_step_warmup_steps": null,
623
+ "optimizer_action_critic": null,
624
+ "optimizer_actor": "adam",
625
+ "optimizer_actor_bc_flow": "adam",
626
+ "optimizer_critic": "adam",
627
+ "optimizer_edit_actor": null,
628
+ "paligemma_tokenizer_path": null,
629
+ "partition_optimizer": true,
630
+ "policy_type": "flow",
631
+ "ppo_batch_size": 256,
632
+ "ppo_lr": 4.5e-05,
633
+ "proprio_dim": null,
634
+ "q_agg": "mean",
635
+ "q_max": 1.0,
636
+ "q_min": 0.0,
637
+ "q_target_agg": "subset_min",
638
+ "q_target_replace": false,
639
+ "q_target_subset": 2,
640
+ "q_variance_reduction": false,
641
+ "q_vr_num_samples": 8,
642
+ "q_vr_reduction": "mean",
643
+ "randn_clip_value": 3.0,
644
+ "shortcut_denoising_steps": 4,
645
+ "shortcut_inference_steps": 4,
646
+ "simba_actor_hidden_dims": [
647
+ 1024,
648
+ 1024
649
+ ],
650
+ "simba_critic_hidden_dims": [
651
+ 1024,
652
+ 1024,
653
+ 1024
654
+ ],
655
+ "subsample_bon": false,
656
+ "target_entropy": null,
657
+ "tau": 0.005,
658
+ "tf_pi_conditioning": "adaln",
659
+ "tf_pi_dropout": 0.0,
660
+ "tf_pi_embed_dim": 256,
661
+ "tf_pi_heads": 4,
662
+ "tf_pi_layers": 8,
663
+ "tf_q_conditioning": "adaln",
664
+ "tf_q_dropout": 0.0,
665
+ "tf_q_embed_dim": 256,
666
+ "tf_q_heads": 4,
667
+ "tf_q_layers": 8,
668
+ "time_embedding": "sinusoidal",
669
+ "time_embedding_dim": 32,
670
+ "train_one_step_in_offline": true,
671
+ "train_one_step_in_online": true,
672
+ "useFiLM": false,
673
+ "useSimBa": false,
674
+ "useTF": false,
675
+ "use_constant_noise": true,
676
+ "use_constant_scheduler_for_bc": true,
677
+ "use_denoiser": false,
678
+ "use_edit_actor": false,
679
+ "use_mip_q": false,
680
+ "use_muon": false,
681
+ "use_one_step_policy": false,
682
+ "use_shortcut": false,
683
+ "use_state": "proprio",
684
+ "use_success_buffer_q": false,
685
+ "value_hidden_dims": [
686
+ 512,
687
+ 512,
688
+ 512
689
+ ],
690
+ "vit_encoder": "minvit",
691
+ "vit_img_proj_dim": 0,
692
+ "vit_state_proj_dim": 0
693
+ },
694
+ "convention": "unified_env_step",
695
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
696
+ "gamma_env": 0.999,
697
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
698
+ "learner": "qc",
699
+ "optimizers": {
700
+ "actor_lr": 0.0001,
701
+ "critic_lr": 0.0003,
702
+ "reset_optimizers_with_lr": "not used",
703
+ "schedule": {
704
+ "actor": 0.0001,
705
+ "critic": 0.0003,
706
+ "decay_end_env_steps": 1606196,
707
+ "min_ratio": 0.1,
708
+ "schedule": "warmup_cosine",
709
+ "temperature": 0.0003,
710
+ "warmup_env_steps": 20000
711
+ }
712
+ },
713
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
714
+ "recipe": {
715
+ "derived": {
716
+ "agent.actor_num_samples": {
717
+ "after": 8,
718
+ "before": 32
719
+ },
720
+ "agent.actor_type": {
721
+ "after": "best-of-n",
722
+ "before": "upstream_yaml"
723
+ },
724
+ "agent.bon_sft": {
725
+ "after": false,
726
+ "before": "upstream_yaml"
727
+ }
728
+ },
729
+ "explicit_overrides": {},
730
+ "source": "third_party/ogpo/scripts/qc/toolhang.sh",
731
+ "upstream_exact": false
732
+ },
733
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
734
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
735
+ "scan_block": 100,
736
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
737
+ "success_terms": {
738
+ "bc_on_success_batch": false,
739
+ "critic_success_td": false,
740
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
741
+ },
742
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
743
+ "unified_notes": {
744
+ "critic_target_every_critic_step": {
745
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
746
+ "value": true
747
+ },
748
+ "success_td_polyak": {
749
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
750
+ "value": true
751
+ }
752
+ },
753
+ "upstream_config": {
754
+ "agent": {
755
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
756
+ "act_max": 3.0794031620025635,
757
+ "act_min": -3.0920298099517822,
758
+ "action_chunking": true,
759
+ "action_critic_hidden_dims": null,
760
+ "action_dim": null,
761
+ "action_num_qs": 10,
762
+ "action_repeat": false,
763
+ "actor_backbone": "mlp",
764
+ "actor_decay_steps": 50000,
765
+ "actor_end_value": 2e-05,
766
+ "actor_hidden_dims": [
767
+ 512,
768
+ 512,
769
+ 512,
770
+ 512,
771
+ 512
772
+ ],
773
+ "actor_layer_norm": false,
774
+ "actor_lr": 0.0003,
775
+ "actor_lr_partition": 0.0001,
776
+ "actor_num_samples": 16,
777
+ "actor_obs": "state",
778
+ "actor_scheduler": "cosine",
779
+ "actor_tau": 0.005,
780
+ "actor_type": "best-of-n",
781
+ "actor_warmup_steps": 2000,
782
+ "actor_weight_decay": 0.0,
783
+ "adv_clip_min": null,
784
+ "agent_name": "acfql",
785
+ "alpha": 500,
786
+ "batch_size": 256,
787
+ "best_of_n": 16,
788
+ "bon_agg": "mean",
789
+ "bon_sft": false,
790
+ "bon_sft_coeff": 1.0,
791
+ "bootstrap_ratio": 0.5,
792
+ "bptt_num_samples": 4,
793
+ "calql_q_subsample": 2,
794
+ "clip_grad_norm": 1000.0,
795
+ "clip_intermediate_actions": true,
796
+ "clip_max_epsilon_multiplier": 1.0,
797
+ "clip_min_epsilon_multiplier": 1.0,
798
+ "constant_noise_std": 0.01,
799
+ "cql_alpha": 5.0,
800
+ "cql_n_actions": 10,
801
+ "cql_temp": 1.0,
802
+ "critic_backbone": "mlp",
803
+ "critic_decay_steps": 5000,
804
+ "critic_end_value": 1e-08,
805
+ "critic_loss_type": "mse",
806
+ "critic_lr": 0.0003,
807
+ "critic_lr_partition": 0.0003,
808
+ "critic_obs": "state",
809
+ "critic_scheduler": "constant",
810
+ "critic_warmup_steps": 500,
811
+ "critic_weight_decay": 1e-05,
812
+ "denoised_clip_value": 1.0,
813
+ "discount": 0.999,
814
+ "edit_action_scale": 0.5,
815
+ "edit_actor_hidden_dims": null,
816
+ "edit_hidden_dims": [
817
+ 256,
818
+ 256,
819
+ 256
820
+ ],
821
+ "edit_init_temperature": 1.0,
822
+ "edit_layer_norm": true,
823
+ "edit_lr": 0.0003,
824
+ "edit_policy_bound": 0.1,
825
+ "edit_start_step": 100000,
826
+ "edit_target_entropy": null,
827
+ "edit_weight_decay": 0.0,
828
+ "encoder": null,
829
+ "entropy_backup": false,
830
+ "flow_steps": 10,
831
+ "fql_distillation_coeff": 10.0,
832
+ "fql_train_main_policy": true,
833
+ "freeze_vision_encoder": false,
834
+ "ft_flow_steps": 10,
835
+ "horizon_length": 8,
836
+ "init_temperature": 1.0,
837
+ "layer_norm": true,
838
+ "lr": 0.0003,
839
+ "lr_action_critic": null,
840
+ "lr_actor": null,
841
+ "lr_actor_bc_flow": null,
842
+ "lr_critic": null,
843
+ "lr_edit_actor": null,
844
+ "lr_noise_critic": null,
845
+ "max_noise_std": 0.01,
846
+ "mc_regression": false,
847
+ "mc_regression_coeff": 1.0,
848
+ "min_noise_std": 0.01,
849
+ "mip_q_ensemble": false,
850
+ "mip_q_noise_scale": 1.0,
851
+ "mip_q_t_star": 0.9,
852
+ "mip_t_star": 0.999,
853
+ "muon_beta": 0.95,
854
+ "muon_nesterov": true,
855
+ "muon_ns_steps": 5,
856
+ "n_action_samples": 8,
857
+ "n_edit_samples": 8,
858
+ "noise_critic_grad_steps": 1,
859
+ "noise_dim": null,
860
+ "noise_net_hidden_dims": [
861
+ 256,
862
+ 256
863
+ ],
864
+ "noise_net_layer_norm": true,
865
+ "normalize_act_space_dimension": false,
866
+ "normalize_denoising_horizon": false,
867
+ "normalize_q_loss": false,
868
+ "num_bins": 256,
869
+ "num_ensemble_members": 10,
870
+ "num_qs": 10,
871
+ "ob_dims": null,
872
+ "one_step_decay_steps": null,
873
+ "one_step_end_value": null,
874
+ "one_step_hidden_dims": [
875
+ 512,
876
+ 512,
877
+ 512,
878
+ 512
879
+ ],
880
+ "one_step_layer_norm": false,
881
+ "one_step_lr": null,
882
+ "one_step_scheduler": null,
883
+ "one_step_warmup_steps": null,
884
+ "optimizer_action_critic": null,
885
+ "optimizer_actor": "adam",
886
+ "optimizer_actor_bc_flow": "adam",
887
+ "optimizer_critic": "adam",
888
+ "optimizer_edit_actor": null,
889
+ "paligemma_tokenizer_path": null,
890
+ "partition_optimizer": true,
891
+ "policy_type": "flow",
892
+ "ppo_batch_size": 256,
893
+ "ppo_lr": 4.5e-05,
894
+ "proprio_dim": null,
895
+ "q_agg": "mean",
896
+ "q_max": null,
897
+ "q_min": null,
898
+ "q_target_agg": "subset_min",
899
+ "q_target_replace": false,
900
+ "q_target_subset": 2,
901
+ "q_variance_reduction": false,
902
+ "q_vr_num_samples": 8,
903
+ "q_vr_reduction": "mean",
904
+ "randn_clip_value": 3.0,
905
+ "shortcut_denoising_steps": 4,
906
+ "shortcut_inference_steps": 4,
907
+ "simba_actor_hidden_dims": [
908
+ 1024,
909
+ 1024
910
+ ],
911
+ "simba_critic_hidden_dims": [
912
+ 1024,
913
+ 1024,
914
+ 1024
915
+ ],
916
+ "subsample_bon": false,
917
+ "target_entropy": null,
918
+ "tau": 0.005,
919
+ "tf_pi_conditioning": "adaln",
920
+ "tf_pi_dropout": 0.0,
921
+ "tf_pi_embed_dim": 256,
922
+ "tf_pi_heads": 4,
923
+ "tf_pi_layers": 8,
924
+ "tf_q_conditioning": "adaln",
925
+ "tf_q_dropout": 0.0,
926
+ "tf_q_embed_dim": 256,
927
+ "tf_q_heads": 4,
928
+ "tf_q_layers": 8,
929
+ "time_embedding": "sinusoidal",
930
+ "time_embedding_dim": 32,
931
+ "train_one_step_in_offline": true,
932
+ "train_one_step_in_online": true,
933
+ "useFiLM": false,
934
+ "useSimBa": false,
935
+ "useTF": false,
936
+ "use_constant_noise": true,
937
+ "use_constant_scheduler_for_bc": true,
938
+ "use_denoiser": false,
939
+ "use_edit_actor": false,
940
+ "use_mip_q": false,
941
+ "use_muon": false,
942
+ "use_one_step_policy": false,
943
+ "use_shortcut": false,
944
+ "use_state": "proprio",
945
+ "use_success_buffer_q": false,
946
+ "value_hidden_dims": [
947
+ 512,
948
+ 512,
949
+ 512
950
+ ],
951
+ "vit_encoder": "minvit",
952
+ "vit_img_proj_dim": 0,
953
+ "vit_state_proj_dim": 0
954
+ },
955
+ "bc": {
956
+ "clip_bc": true,
957
+ "clip_bc_threshold": 0.45,
958
+ "clip_bc_wrt": "ODE"
959
+ },
960
+ "buffer": {
961
+ "offline_ratio": 0.0,
962
+ "size": 2000000,
963
+ "start_training": 10000,
964
+ "utd_online": 1,
965
+ "utd_pi": null,
966
+ "utd_q": null,
967
+ "utd_v": null,
968
+ "utd_warmup": 1
969
+ },
970
+ "checkpoint": {
971
+ "ep_resume": 0,
972
+ "restore_actor_path": null,
973
+ "restore_critic_path": null,
974
+ "restore_encoder_path": null
975
+ },
976
+ "dataset": {
977
+ "num_offline_trajs": -1,
978
+ "ogbench_dir": null,
979
+ "p_aug": 0.0,
980
+ "proportion": 1.0,
981
+ "replace_interval": 1000
982
+ },
983
+ "env": {
984
+ "name": "tool_hang-ph-low_dim",
985
+ "post_success_steps": 0,
986
+ "rew_fn": "sparse",
987
+ "sparse": false
988
+ },
989
+ "eval": {
990
+ "eval_episodes": 2,
991
+ "n_eval_envs": 32,
992
+ "plot_q_vs_mc": false,
993
+ "video_episodes": 0,
994
+ "video_frame_skip": 3
995
+ },
996
+ "experiment": {
997
+ "log": true,
998
+ "project": "QC",
999
+ "run_group": "toolhang",
1000
+ "run_name": "qc_toolhang",
1001
+ "save_dir": "exp/ogpo",
1002
+ "seed": 1,
1003
+ "wandb_name": "qc_toolhang"
1004
+ },
1005
+ "intervals": {
1006
+ "eval_interval": 20000,
1007
+ "eval_interval_bc": 5000,
1008
+ "log_interval": 5000,
1009
+ "preemption_backup_interval": -1,
1010
+ "save_interval": 500000,
1011
+ "save_interval_bc": -1,
1012
+ "step_data_log_interval": -1
1013
+ },
1014
+ "success_buffer": {
1015
+ "batch_size": 256,
1016
+ "use_success_buffer": false
1017
+ },
1018
+ "training": {
1019
+ "bc_only": false,
1020
+ "bc_pi_steps": 0,
1021
+ "bc_q_steps": 0,
1022
+ "bc_refine_steps": 0,
1023
+ "calql_steps": 0,
1024
+ "force_jax_sync": true,
1025
+ "gpu_mode": "single",
1026
+ "online_steps": 5000000,
1027
+ "q_warmup_steps": 0
1028
+ }
1029
+ },
1030
+ "upstream_source": {
1031
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1032
+ "patches_sha256": {
1033
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1034
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1035
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1036
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1037
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1038
+ }
1039
+ }
1040
+ },
1041
+ "observation_mode": "pixel_state",
1042
+ "offline_buffer": {
1043
+ "actions_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1044
+ "aux_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1045
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
1046
+ "chunks": 5946,
1047
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1048
+ "decision_ref": "[1017-S2OFFBC]",
1049
+ "env_steps": 47168,
1050
+ "features_sha256": "7af4a3aee67b4a7eae0c45d07974462977e71712cd76bb9afd89dfdb9130699b",
1051
+ "horizon": 8,
1052
+ "identity_hash": "d54f7fa39190225c48c52a3782ea8d00b6d59a0a08d5c6b085b1a6af164eff34",
1053
+ "learner_coordinates": "JaxStudy2Learner:qc",
1054
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1055
+ "ratio": {
1056
+ "adaptive_ratio": false,
1057
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1058
+ },
1059
+ "selection": {
1060
+ "cohort_size": 100,
1061
+ "demos": [
1062
+ "demo_0",
1063
+ "demo_5",
1064
+ "demo_6",
1065
+ "demo_7",
1066
+ "demo_9",
1067
+ "demo_12",
1068
+ "demo_16",
1069
+ "demo_17",
1070
+ "demo_18",
1071
+ "demo_20",
1072
+ "demo_25",
1073
+ "demo_27",
1074
+ "demo_30",
1075
+ "demo_31",
1076
+ "demo_32",
1077
+ "demo_35",
1078
+ "demo_37",
1079
+ "demo_38",
1080
+ "demo_43",
1081
+ "demo_44",
1082
+ "demo_45",
1083
+ "demo_46",
1084
+ "demo_50",
1085
+ "demo_51",
1086
+ "demo_55",
1087
+ "demo_56",
1088
+ "demo_57",
1089
+ "demo_58",
1090
+ "demo_69",
1091
+ "demo_70",
1092
+ "demo_71",
1093
+ "demo_72",
1094
+ "demo_74",
1095
+ "demo_78",
1096
+ "demo_79",
1097
+ "demo_81",
1098
+ "demo_82",
1099
+ "demo_84",
1100
+ "demo_85",
1101
+ "demo_86",
1102
+ "demo_90",
1103
+ "demo_91",
1104
+ "demo_92",
1105
+ "demo_95",
1106
+ "demo_96",
1107
+ "demo_97",
1108
+ "demo_98",
1109
+ "demo_101",
1110
+ "demo_104",
1111
+ "demo_107",
1112
+ "demo_111",
1113
+ "demo_112",
1114
+ "demo_113",
1115
+ "demo_115",
1116
+ "demo_116",
1117
+ "demo_119",
1118
+ "demo_121",
1119
+ "demo_123",
1120
+ "demo_125",
1121
+ "demo_127",
1122
+ "demo_128",
1123
+ "demo_131",
1124
+ "demo_132",
1125
+ "demo_133",
1126
+ "demo_134",
1127
+ "demo_137",
1128
+ "demo_138",
1129
+ "demo_140",
1130
+ "demo_141",
1131
+ "demo_142",
1132
+ "demo_145",
1133
+ "demo_149",
1134
+ "demo_150",
1135
+ "demo_151",
1136
+ "demo_154",
1137
+ "demo_157",
1138
+ "demo_159",
1139
+ "demo_161",
1140
+ "demo_164",
1141
+ "demo_165",
1142
+ "demo_169",
1143
+ "demo_171",
1144
+ "demo_172",
1145
+ "demo_173",
1146
+ "demo_175",
1147
+ "demo_176",
1148
+ "demo_177",
1149
+ "demo_178",
1150
+ "demo_179",
1151
+ "demo_181",
1152
+ "demo_182",
1153
+ "demo_183",
1154
+ "demo_184",
1155
+ "demo_185",
1156
+ "demo_187",
1157
+ "demo_188",
1158
+ "demo_191",
1159
+ "demo_192",
1160
+ "demo_193",
1161
+ "demo_196"
1162
+ ],
1163
+ "manifest": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/source_input/data_manifest.json",
1164
+ "manifest_sha256": "6bcfaec343fc97c766bacf650dc6a73a71c0c9969974d21bfbfd1c1ac113685f",
1165
+ "regime": "mixed100",
1166
+ "source": "bc_regime",
1167
+ "source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1168
+ "split": {
1169
+ "training": 90,
1170
+ "validation": 10
1171
+ },
1172
+ "task": "tool_hang"
1173
+ }
1174
+ },
1175
+ "offline_role": "mixed",
1176
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1177
+ "smoke": false,
1178
+ "task": "tool_hang",
1179
+ "task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
1180
+ },
1181
+ "seed": 1,
1182
+ "start_policy": "base_cfm_ode",
1183
+ "variant": {
1184
+ "flags": {
1185
+ "bon_sft": false,
1186
+ "use_success_buffer": false
1187
+ },
1188
+ "learner": "qc",
1189
+ "name": "qc",
1190
+ "phase": 1,
1191
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1192
+ "version": "study2_variants_v4"
1193
+ },
1194
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1195
+ }
RL/ToolHang100/QC/mixed100/seed_1/final/state.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70dda4b58013596119a7b3bbb3d85a2f871fba438451a99f2754c18290928ead
3
+ size 207224453
RL/ToolHang100/QC/mixed100/seed_1/frozen_base.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "after": "848c66c08785dca4f652c3a7eb0bd2cb26761fbdf91fb1dbcc2cde5be0446931",
3
+ "before": "848c66c08785dca4f652c3a7eb0bd2cb26761fbdf91fb1dbcc2cde5be0446931",
4
+ "checkpoint_unchanged": true,
5
+ "unchanged": true
6
+ }
RL/ToolHang100/QC/mixed100/seed_1/offline_buffer.json ADDED
@@ -0,0 +1,847 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "identity": {
3
+ "actions_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
4
+ "aux_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
5
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
6
+ "chunks": 5946,
7
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
8
+ "decision_ref": "[1017-S2OFFBC]",
9
+ "env_steps": 47168,
10
+ "features_sha256": "7af4a3aee67b4a7eae0c45d07974462977e71712cd76bb9afd89dfdb9130699b",
11
+ "horizon": 8,
12
+ "learner_coordinates": "JaxStudy2Learner:qc",
13
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
14
+ "ratio": {
15
+ "adaptive_ratio": false,
16
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
17
+ },
18
+ "selection": {
19
+ "cohort_size": 100,
20
+ "demos": [
21
+ "demo_0",
22
+ "demo_5",
23
+ "demo_6",
24
+ "demo_7",
25
+ "demo_9",
26
+ "demo_12",
27
+ "demo_16",
28
+ "demo_17",
29
+ "demo_18",
30
+ "demo_20",
31
+ "demo_25",
32
+ "demo_27",
33
+ "demo_30",
34
+ "demo_31",
35
+ "demo_32",
36
+ "demo_35",
37
+ "demo_37",
38
+ "demo_38",
39
+ "demo_43",
40
+ "demo_44",
41
+ "demo_45",
42
+ "demo_46",
43
+ "demo_50",
44
+ "demo_51",
45
+ "demo_55",
46
+ "demo_56",
47
+ "demo_57",
48
+ "demo_58",
49
+ "demo_69",
50
+ "demo_70",
51
+ "demo_71",
52
+ "demo_72",
53
+ "demo_74",
54
+ "demo_78",
55
+ "demo_79",
56
+ "demo_81",
57
+ "demo_82",
58
+ "demo_84",
59
+ "demo_85",
60
+ "demo_86",
61
+ "demo_90",
62
+ "demo_91",
63
+ "demo_92",
64
+ "demo_95",
65
+ "demo_96",
66
+ "demo_97",
67
+ "demo_98",
68
+ "demo_101",
69
+ "demo_104",
70
+ "demo_107",
71
+ "demo_111",
72
+ "demo_112",
73
+ "demo_113",
74
+ "demo_115",
75
+ "demo_116",
76
+ "demo_119",
77
+ "demo_121",
78
+ "demo_123",
79
+ "demo_125",
80
+ "demo_127",
81
+ "demo_128",
82
+ "demo_131",
83
+ "demo_132",
84
+ "demo_133",
85
+ "demo_134",
86
+ "demo_137",
87
+ "demo_138",
88
+ "demo_140",
89
+ "demo_141",
90
+ "demo_142",
91
+ "demo_145",
92
+ "demo_149",
93
+ "demo_150",
94
+ "demo_151",
95
+ "demo_154",
96
+ "demo_157",
97
+ "demo_159",
98
+ "demo_161",
99
+ "demo_164",
100
+ "demo_165",
101
+ "demo_169",
102
+ "demo_171",
103
+ "demo_172",
104
+ "demo_173",
105
+ "demo_175",
106
+ "demo_176",
107
+ "demo_177",
108
+ "demo_178",
109
+ "demo_179",
110
+ "demo_181",
111
+ "demo_182",
112
+ "demo_183",
113
+ "demo_184",
114
+ "demo_185",
115
+ "demo_187",
116
+ "demo_188",
117
+ "demo_191",
118
+ "demo_192",
119
+ "demo_193",
120
+ "demo_196"
121
+ ],
122
+ "manifest": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/source_input/data_manifest.json",
123
+ "manifest_sha256": "6bcfaec343fc97c766bacf650dc6a73a71c0c9969974d21bfbfd1c1ac113685f",
124
+ "regime": "mixed100",
125
+ "source": "bc_regime",
126
+ "source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
127
+ "split": {
128
+ "training": 90,
129
+ "validation": 10
130
+ },
131
+ "task": "tool_hang"
132
+ }
133
+ },
134
+ "identity_hash": "d54f7fa39190225c48c52a3782ea8d00b6d59a0a08d5c6b085b1a6af164eff34",
135
+ "stats": {
136
+ "chunks": 5946,
137
+ "dataset": "/home/nhy/RL-postTrain/repo/data/robomimic/tool_hang/ph/low_dim_v15.hdf5",
138
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
139
+ "env_steps": 47168,
140
+ "per_demo": [
141
+ {
142
+ "chunks": 85,
143
+ "demo": "demo_0",
144
+ "env_steps": 677,
145
+ "frames": 681,
146
+ "terminal_next_frame": 677
147
+ },
148
+ {
149
+ "chunks": 85,
150
+ "demo": "demo_5",
151
+ "env_steps": 677,
152
+ "frames": 681,
153
+ "terminal_next_frame": 677
154
+ },
155
+ {
156
+ "chunks": 69,
157
+ "demo": "demo_6",
158
+ "env_steps": 551,
159
+ "frames": 587,
160
+ "terminal_next_frame": 551
161
+ },
162
+ {
163
+ "chunks": 78,
164
+ "demo": "demo_7",
165
+ "env_steps": 618,
166
+ "frames": 622,
167
+ "terminal_next_frame": 618
168
+ },
169
+ {
170
+ "chunks": 78,
171
+ "demo": "demo_9",
172
+ "env_steps": 619,
173
+ "frames": 623,
174
+ "terminal_next_frame": 619
175
+ },
176
+ {
177
+ "chunks": 62,
178
+ "demo": "demo_12",
179
+ "env_steps": 492,
180
+ "frames": 496,
181
+ "terminal_next_frame": 492
182
+ },
183
+ {
184
+ "chunks": 84,
185
+ "demo": "demo_16",
186
+ "env_steps": 665,
187
+ "frames": 676,
188
+ "terminal_next_frame": 665
189
+ },
190
+ {
191
+ "chunks": 84,
192
+ "demo": "demo_17",
193
+ "env_steps": 668,
194
+ "frames": 677,
195
+ "terminal_next_frame": 668
196
+ },
197
+ {
198
+ "chunks": 76,
199
+ "demo": "demo_18",
200
+ "env_steps": 605,
201
+ "frames": 632,
202
+ "terminal_next_frame": 605
203
+ },
204
+ {
205
+ "chunks": 71,
206
+ "demo": "demo_20",
207
+ "env_steps": 561,
208
+ "frames": 565,
209
+ "terminal_next_frame": 561
210
+ },
211
+ {
212
+ "chunks": 60,
213
+ "demo": "demo_25",
214
+ "env_steps": 475,
215
+ "frames": 480,
216
+ "terminal_next_frame": 475
217
+ },
218
+ {
219
+ "chunks": 59,
220
+ "demo": "demo_27",
221
+ "env_steps": 465,
222
+ "frames": 469,
223
+ "terminal_next_frame": 465
224
+ },
225
+ {
226
+ "chunks": 51,
227
+ "demo": "demo_30",
228
+ "env_steps": 405,
229
+ "frames": 409,
230
+ "terminal_next_frame": 405
231
+ },
232
+ {
233
+ "chunks": 58,
234
+ "demo": "demo_31",
235
+ "env_steps": 458,
236
+ "frames": 476,
237
+ "terminal_next_frame": 458
238
+ },
239
+ {
240
+ "chunks": 59,
241
+ "demo": "demo_32",
242
+ "env_steps": 467,
243
+ "frames": 473,
244
+ "terminal_next_frame": 467
245
+ },
246
+ {
247
+ "chunks": 74,
248
+ "demo": "demo_35",
249
+ "env_steps": 585,
250
+ "frames": 589,
251
+ "terminal_next_frame": 585
252
+ },
253
+ {
254
+ "chunks": 71,
255
+ "demo": "demo_37",
256
+ "env_steps": 563,
257
+ "frames": 567,
258
+ "terminal_next_frame": 563
259
+ },
260
+ {
261
+ "chunks": 59,
262
+ "demo": "demo_38",
263
+ "env_steps": 466,
264
+ "frames": 471,
265
+ "terminal_next_frame": 466
266
+ },
267
+ {
268
+ "chunks": 62,
269
+ "demo": "demo_43",
270
+ "env_steps": 491,
271
+ "frames": 495,
272
+ "terminal_next_frame": 491
273
+ },
274
+ {
275
+ "chunks": 65,
276
+ "demo": "demo_44",
277
+ "env_steps": 516,
278
+ "frames": 523,
279
+ "terminal_next_frame": 516
280
+ },
281
+ {
282
+ "chunks": 78,
283
+ "demo": "demo_45",
284
+ "env_steps": 623,
285
+ "frames": 627,
286
+ "terminal_next_frame": 623
287
+ },
288
+ {
289
+ "chunks": 56,
290
+ "demo": "demo_46",
291
+ "env_steps": 446,
292
+ "frames": 450,
293
+ "terminal_next_frame": 446
294
+ },
295
+ {
296
+ "chunks": 54,
297
+ "demo": "demo_50",
298
+ "env_steps": 432,
299
+ "frames": 509,
300
+ "terminal_next_frame": 432
301
+ },
302
+ {
303
+ "chunks": 63,
304
+ "demo": "demo_51",
305
+ "env_steps": 498,
306
+ "frames": 502,
307
+ "terminal_next_frame": 498
308
+ },
309
+ {
310
+ "chunks": 65,
311
+ "demo": "demo_55",
312
+ "env_steps": 517,
313
+ "frames": 521,
314
+ "terminal_next_frame": 517
315
+ },
316
+ {
317
+ "chunks": 59,
318
+ "demo": "demo_56",
319
+ "env_steps": 465,
320
+ "frames": 469,
321
+ "terminal_next_frame": 465
322
+ },
323
+ {
324
+ "chunks": 59,
325
+ "demo": "demo_57",
326
+ "env_steps": 472,
327
+ "frames": 476,
328
+ "terminal_next_frame": 472
329
+ },
330
+ {
331
+ "chunks": 54,
332
+ "demo": "demo_58",
333
+ "env_steps": 429,
334
+ "frames": 433,
335
+ "terminal_next_frame": 429
336
+ },
337
+ {
338
+ "chunks": 68,
339
+ "demo": "demo_69",
340
+ "env_steps": 538,
341
+ "frames": 542,
342
+ "terminal_next_frame": 538
343
+ },
344
+ {
345
+ "chunks": 53,
346
+ "demo": "demo_70",
347
+ "env_steps": 417,
348
+ "frames": 421,
349
+ "terminal_next_frame": 417
350
+ },
351
+ {
352
+ "chunks": 70,
353
+ "demo": "demo_71",
354
+ "env_steps": 555,
355
+ "frames": 562,
356
+ "terminal_next_frame": 555
357
+ },
358
+ {
359
+ "chunks": 52,
360
+ "demo": "demo_72",
361
+ "env_steps": 412,
362
+ "frames": 416,
363
+ "terminal_next_frame": 412
364
+ },
365
+ {
366
+ "chunks": 59,
367
+ "demo": "demo_74",
368
+ "env_steps": 468,
369
+ "frames": 470,
370
+ "terminal_next_frame": 468
371
+ },
372
+ {
373
+ "chunks": 56,
374
+ "demo": "demo_78",
375
+ "env_steps": 448,
376
+ "frames": 452,
377
+ "terminal_next_frame": 448
378
+ },
379
+ {
380
+ "chunks": 52,
381
+ "demo": "demo_79",
382
+ "env_steps": 410,
383
+ "frames": 414,
384
+ "terminal_next_frame": 410
385
+ },
386
+ {
387
+ "chunks": 45,
388
+ "demo": "demo_81",
389
+ "env_steps": 359,
390
+ "frames": 363,
391
+ "terminal_next_frame": 359
392
+ },
393
+ {
394
+ "chunks": 52,
395
+ "demo": "demo_82",
396
+ "env_steps": 416,
397
+ "frames": 422,
398
+ "terminal_next_frame": 416
399
+ },
400
+ {
401
+ "chunks": 54,
402
+ "demo": "demo_84",
403
+ "env_steps": 426,
404
+ "frames": 430,
405
+ "terminal_next_frame": 426
406
+ },
407
+ {
408
+ "chunks": 61,
409
+ "demo": "demo_85",
410
+ "env_steps": 484,
411
+ "frames": 488,
412
+ "terminal_next_frame": 484
413
+ },
414
+ {
415
+ "chunks": 60,
416
+ "demo": "demo_86",
417
+ "env_steps": 475,
418
+ "frames": 479,
419
+ "terminal_next_frame": 475
420
+ },
421
+ {
422
+ "chunks": 51,
423
+ "demo": "demo_90",
424
+ "env_steps": 406,
425
+ "frames": 410,
426
+ "terminal_next_frame": 406
427
+ },
428
+ {
429
+ "chunks": 53,
430
+ "demo": "demo_91",
431
+ "env_steps": 421,
432
+ "frames": 446,
433
+ "terminal_next_frame": 421
434
+ },
435
+ {
436
+ "chunks": 46,
437
+ "demo": "demo_92",
438
+ "env_steps": 365,
439
+ "frames": 369,
440
+ "terminal_next_frame": 365
441
+ },
442
+ {
443
+ "chunks": 62,
444
+ "demo": "demo_95",
445
+ "env_steps": 490,
446
+ "frames": 496,
447
+ "terminal_next_frame": 490
448
+ },
449
+ {
450
+ "chunks": 65,
451
+ "demo": "demo_96",
452
+ "env_steps": 515,
453
+ "frames": 523,
454
+ "terminal_next_frame": 515
455
+ },
456
+ {
457
+ "chunks": 76,
458
+ "demo": "demo_97",
459
+ "env_steps": 601,
460
+ "frames": 608,
461
+ "terminal_next_frame": 601
462
+ },
463
+ {
464
+ "chunks": 49,
465
+ "demo": "demo_98",
466
+ "env_steps": 390,
467
+ "frames": 394,
468
+ "terminal_next_frame": 390
469
+ },
470
+ {
471
+ "chunks": 52,
472
+ "demo": "demo_101",
473
+ "env_steps": 410,
474
+ "frames": 414,
475
+ "terminal_next_frame": 410
476
+ },
477
+ {
478
+ "chunks": 50,
479
+ "demo": "demo_104",
480
+ "env_steps": 398,
481
+ "frames": 402,
482
+ "terminal_next_frame": 398
483
+ },
484
+ {
485
+ "chunks": 47,
486
+ "demo": "demo_107",
487
+ "env_steps": 371,
488
+ "frames": 375,
489
+ "terminal_next_frame": 371
490
+ },
491
+ {
492
+ "chunks": 51,
493
+ "demo": "demo_111",
494
+ "env_steps": 401,
495
+ "frames": 405,
496
+ "terminal_next_frame": 401
497
+ },
498
+ {
499
+ "chunks": 61,
500
+ "demo": "demo_112",
501
+ "env_steps": 482,
502
+ "frames": 486,
503
+ "terminal_next_frame": 482
504
+ },
505
+ {
506
+ "chunks": 55,
507
+ "demo": "demo_113",
508
+ "env_steps": 436,
509
+ "frames": 440,
510
+ "terminal_next_frame": 436
511
+ },
512
+ {
513
+ "chunks": 59,
514
+ "demo": "demo_115",
515
+ "env_steps": 466,
516
+ "frames": 470,
517
+ "terminal_next_frame": 466
518
+ },
519
+ {
520
+ "chunks": 55,
521
+ "demo": "demo_116",
522
+ "env_steps": 434,
523
+ "frames": 472,
524
+ "terminal_next_frame": 434
525
+ },
526
+ {
527
+ "chunks": 67,
528
+ "demo": "demo_119",
529
+ "env_steps": 531,
530
+ "frames": 535,
531
+ "terminal_next_frame": 531
532
+ },
533
+ {
534
+ "chunks": 47,
535
+ "demo": "demo_121",
536
+ "env_steps": 371,
537
+ "frames": 375,
538
+ "terminal_next_frame": 371
539
+ },
540
+ {
541
+ "chunks": 48,
542
+ "demo": "demo_123",
543
+ "env_steps": 383,
544
+ "frames": 415,
545
+ "terminal_next_frame": 383
546
+ },
547
+ {
548
+ "chunks": 52,
549
+ "demo": "demo_125",
550
+ "env_steps": 410,
551
+ "frames": 414,
552
+ "terminal_next_frame": 410
553
+ },
554
+ {
555
+ "chunks": 50,
556
+ "demo": "demo_127",
557
+ "env_steps": 396,
558
+ "frames": 421,
559
+ "terminal_next_frame": 396
560
+ },
561
+ {
562
+ "chunks": 71,
563
+ "demo": "demo_128",
564
+ "env_steps": 563,
565
+ "frames": 567,
566
+ "terminal_next_frame": 563
567
+ },
568
+ {
569
+ "chunks": 42,
570
+ "demo": "demo_131",
571
+ "env_steps": 332,
572
+ "frames": 336,
573
+ "terminal_next_frame": 332
574
+ },
575
+ {
576
+ "chunks": 54,
577
+ "demo": "demo_132",
578
+ "env_steps": 427,
579
+ "frames": 431,
580
+ "terminal_next_frame": 427
581
+ },
582
+ {
583
+ "chunks": 62,
584
+ "demo": "demo_133",
585
+ "env_steps": 494,
586
+ "frames": 498,
587
+ "terminal_next_frame": 494
588
+ },
589
+ {
590
+ "chunks": 41,
591
+ "demo": "demo_134",
592
+ "env_steps": 328,
593
+ "frames": 332,
594
+ "terminal_next_frame": 328
595
+ },
596
+ {
597
+ "chunks": 50,
598
+ "demo": "demo_137",
599
+ "env_steps": 400,
600
+ "frames": 406,
601
+ "terminal_next_frame": 400
602
+ },
603
+ {
604
+ "chunks": 70,
605
+ "demo": "demo_138",
606
+ "env_steps": 554,
607
+ "frames": 580,
608
+ "terminal_next_frame": 554
609
+ },
610
+ {
611
+ "chunks": 71,
612
+ "demo": "demo_140",
613
+ "env_steps": 568,
614
+ "frames": 572,
615
+ "terminal_next_frame": 568
616
+ },
617
+ {
618
+ "chunks": 45,
619
+ "demo": "demo_141",
620
+ "env_steps": 357,
621
+ "frames": 361,
622
+ "terminal_next_frame": 357
623
+ },
624
+ {
625
+ "chunks": 56,
626
+ "demo": "demo_142",
627
+ "env_steps": 442,
628
+ "frames": 446,
629
+ "terminal_next_frame": 442
630
+ },
631
+ {
632
+ "chunks": 48,
633
+ "demo": "demo_145",
634
+ "env_steps": 379,
635
+ "frames": 384,
636
+ "terminal_next_frame": 379
637
+ },
638
+ {
639
+ "chunks": 59,
640
+ "demo": "demo_149",
641
+ "env_steps": 466,
642
+ "frames": 474,
643
+ "terminal_next_frame": 466
644
+ },
645
+ {
646
+ "chunks": 45,
647
+ "demo": "demo_150",
648
+ "env_steps": 358,
649
+ "frames": 362,
650
+ "terminal_next_frame": 358
651
+ },
652
+ {
653
+ "chunks": 87,
654
+ "demo": "demo_151",
655
+ "env_steps": 694,
656
+ "frames": 698,
657
+ "terminal_next_frame": 694
658
+ },
659
+ {
660
+ "chunks": 50,
661
+ "demo": "demo_154",
662
+ "env_steps": 393,
663
+ "frames": 397,
664
+ "terminal_next_frame": 393
665
+ },
666
+ {
667
+ "chunks": 55,
668
+ "demo": "demo_157",
669
+ "env_steps": 433,
670
+ "frames": 437,
671
+ "terminal_next_frame": 433
672
+ },
673
+ {
674
+ "chunks": 57,
675
+ "demo": "demo_159",
676
+ "env_steps": 456,
677
+ "frames": 465,
678
+ "terminal_next_frame": 456
679
+ },
680
+ {
681
+ "chunks": 53,
682
+ "demo": "demo_161",
683
+ "env_steps": 421,
684
+ "frames": 425,
685
+ "terminal_next_frame": 421
686
+ },
687
+ {
688
+ "chunks": 67,
689
+ "demo": "demo_164",
690
+ "env_steps": 529,
691
+ "frames": 533,
692
+ "terminal_next_frame": 529
693
+ },
694
+ {
695
+ "chunks": 63,
696
+ "demo": "demo_165",
697
+ "env_steps": 503,
698
+ "frames": 530,
699
+ "terminal_next_frame": 503
700
+ },
701
+ {
702
+ "chunks": 44,
703
+ "demo": "demo_169",
704
+ "env_steps": 349,
705
+ "frames": 353,
706
+ "terminal_next_frame": 349
707
+ },
708
+ {
709
+ "chunks": 68,
710
+ "demo": "demo_171",
711
+ "env_steps": 537,
712
+ "frames": 541,
713
+ "terminal_next_frame": 537
714
+ },
715
+ {
716
+ "chunks": 69,
717
+ "demo": "demo_172",
718
+ "env_steps": 550,
719
+ "frames": 554,
720
+ "terminal_next_frame": 550
721
+ },
722
+ {
723
+ "chunks": 54,
724
+ "demo": "demo_173",
725
+ "env_steps": 432,
726
+ "frames": 441,
727
+ "terminal_next_frame": 432
728
+ },
729
+ {
730
+ "chunks": 65,
731
+ "demo": "demo_175",
732
+ "env_steps": 515,
733
+ "frames": 525,
734
+ "terminal_next_frame": 515
735
+ },
736
+ {
737
+ "chunks": 49,
738
+ "demo": "demo_176",
739
+ "env_steps": 391,
740
+ "frames": 395,
741
+ "terminal_next_frame": 391
742
+ },
743
+ {
744
+ "chunks": 54,
745
+ "demo": "demo_177",
746
+ "env_steps": 429,
747
+ "frames": 433,
748
+ "terminal_next_frame": 429
749
+ },
750
+ {
751
+ "chunks": 48,
752
+ "demo": "demo_178",
753
+ "env_steps": 380,
754
+ "frames": 384,
755
+ "terminal_next_frame": 380
756
+ },
757
+ {
758
+ "chunks": 57,
759
+ "demo": "demo_179",
760
+ "env_steps": 449,
761
+ "frames": 454,
762
+ "terminal_next_frame": 449
763
+ },
764
+ {
765
+ "chunks": 53,
766
+ "demo": "demo_181",
767
+ "env_steps": 422,
768
+ "frames": 426,
769
+ "terminal_next_frame": 422
770
+ },
771
+ {
772
+ "chunks": 73,
773
+ "demo": "demo_182",
774
+ "env_steps": 582,
775
+ "frames": 586,
776
+ "terminal_next_frame": 582
777
+ },
778
+ {
779
+ "chunks": 56,
780
+ "demo": "demo_183",
781
+ "env_steps": 443,
782
+ "frames": 447,
783
+ "terminal_next_frame": 443
784
+ },
785
+ {
786
+ "chunks": 50,
787
+ "demo": "demo_184",
788
+ "env_steps": 395,
789
+ "frames": 399,
790
+ "terminal_next_frame": 395
791
+ },
792
+ {
793
+ "chunks": 46,
794
+ "demo": "demo_185",
795
+ "env_steps": 365,
796
+ "frames": 369,
797
+ "terminal_next_frame": 365
798
+ },
799
+ {
800
+ "chunks": 50,
801
+ "demo": "demo_187",
802
+ "env_steps": 399,
803
+ "frames": 420,
804
+ "terminal_next_frame": 399
805
+ },
806
+ {
807
+ "chunks": 89,
808
+ "demo": "demo_188",
809
+ "env_steps": 705,
810
+ "frames": 709,
811
+ "terminal_next_frame": 705
812
+ },
813
+ {
814
+ "chunks": 52,
815
+ "demo": "demo_191",
816
+ "env_steps": 411,
817
+ "frames": 415,
818
+ "terminal_next_frame": 411
819
+ },
820
+ {
821
+ "chunks": 52,
822
+ "demo": "demo_192",
823
+ "env_steps": 413,
824
+ "frames": 416,
825
+ "terminal_next_frame": 413
826
+ },
827
+ {
828
+ "chunks": 62,
829
+ "demo": "demo_193",
830
+ "env_steps": 494,
831
+ "frames": 498,
832
+ "terminal_next_frame": 494
833
+ },
834
+ {
835
+ "chunks": 53,
836
+ "demo": "demo_196",
837
+ "env_steps": 419,
838
+ "frames": 423,
839
+ "terminal_next_frame": 419
840
+ }
841
+ ],
842
+ "pixel_cache": "/home/nhy/RL-postTrain/repo/data/pixels/tool_hang/v1",
843
+ "raw_coords": 330176,
844
+ "raw_coords_clipped": 0
845
+ },
846
+ "stats_hash": "b75a106510f41a44d458249460de39e45c3c3b9d9f30db5f5826bc7aa64a21cd"
847
+ }
RL/ToolHang100/QC/mixed100/seed_1/publication.json ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "ToolHang100",
3
+ "environment_task": "tool_hang",
4
+ "algorithm": "QC",
5
+ "variant": "qc",
6
+ "cohort": "mixed100",
7
+ "seed": 1,
8
+ "phase": 1,
9
+ "protocol_version": "study2_protocol_v10",
10
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
11
+ "status": "budget_complete",
12
+ "source_host": "HRG2",
13
+ "source_run": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1",
14
+ "path": "RL/ToolHang100/QC/mixed100/seed_1",
15
+ "checkpoints": {
16
+ "final": {
17
+ "path": "RL/ToolHang100/QC/mixed100/seed_1/final/state.pkl",
18
+ "sha256": "70dda4b58013596119a7b3bbb3d85a2f871fba438451a99f2754c18290928ead",
19
+ "bytes": 207224453,
20
+ "source_path": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoints/final/session_1792988385165681572/checkpoint/state.pkl",
21
+ "evaluation": {
22
+ "episodes": 300,
23
+ "imputed": false,
24
+ "n_success": 70,
25
+ "observed": true,
26
+ "seed_start": 800000,
27
+ "sr": 0.23333333333333334
28
+ }
29
+ },
30
+ "best": {
31
+ "path": "RL/ToolHang100/QC/mixed100/seed_1/best/state.pkl",
32
+ "sha256": "6ddafc0ff69f6fdea7c2951cad4b6ba09f3e32375fd411f9cd7abbec81adb8d9",
33
+ "bytes": 207205195,
34
+ "source_path": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoints/best/eval_00007_step_143879_session_1792903293409232482/checkpoint/state.pkl",
35
+ "evaluation": {
36
+ "chunk_transitions": 18000,
37
+ "episodes": 200,
38
+ "eval_index": 7,
39
+ "gain": null,
40
+ "gradient_updates": 54000,
41
+ "ma": null,
42
+ "n_success": 66,
43
+ "policy_version": 18,
44
+ "rl_env_steps": 143879,
45
+ "round": 18,
46
+ "seed_start": 960000,
47
+ "sr": 0.33,
48
+ "success_requested": 0,
49
+ "success_requested_imitation": 0,
50
+ "success_rounds_with_skips": 0,
51
+ "success_skipped": 0,
52
+ "success_skipped_imitation": 0,
53
+ "success_skipped_td": 0,
54
+ "target_step": 140000,
55
+ "update_index": 54000
56
+ }
57
+ }
58
+ },
59
+ "base_bc": {
60
+ "repo_id": "yangfy0627/RL-postTrain",
61
+ "revision": "95320f4b68a221fc235e2897a21558fa43545015",
62
+ "path": "Robomimic/study1/ToolHang/mixed100/seed_1/checkpoints/epoch_300.pkl",
63
+ "sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8"
64
+ },
65
+ "export_scope": "learner state checkpoints, original configuration, provenance and evaluation records; replay segments and offline_buffer.pkl remain in source runs",
66
+ "best_selection": "maximum observed periodic success rate on 200 episodes, earliest tie",
67
+ "prepared_at": "2026-10-26T20:55:55.938473+00:00"
68
+ }
RL/ToolHang100/QC/mixed100/seed_1/resolved_config.json ADDED
@@ -0,0 +1,1195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
3
+ "env": {
4
+ "chunk_length": 8,
5
+ "n_envs": 4
6
+ },
7
+ "learner": {
8
+ "convention": "unified_env_step",
9
+ "gamma_env": 0.999
10
+ },
11
+ "overrides": [
12
+ {
13
+ "decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
14
+ "key": "n_step_chunks",
15
+ "reason": "task-specific unified Study2 setting",
16
+ "upstream": 3,
17
+ "value": 5
18
+ },
19
+ {
20
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
21
+ "key": "offline.cohort_size",
22
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
23
+ "upstream": 50,
24
+ "value": 100
25
+ },
26
+ {
27
+ "decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
28
+ "key": "offline.episodes",
29
+ "reason": "tool_hang regime mixed100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
30
+ "upstream": 50,
31
+ "value": 100
32
+ },
33
+ {
34
+ "decision_ref": "[0930-S2DEC]",
35
+ "key": "learner.qc.agent.act_bounds_json",
36
+ "reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
37
+ "upstream": null,
38
+ "value": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}"
39
+ },
40
+ {
41
+ "decision_ref": "[0930-S2DEC]",
42
+ "key": "learner.qc.agent.act_max",
43
+ "reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
44
+ "upstream": 1.0,
45
+ "value": 3.0794031620025635
46
+ },
47
+ {
48
+ "decision_ref": "[0930-S2DEC]",
49
+ "key": "learner.qc.agent.act_min",
50
+ "reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
51
+ "upstream": -1.0,
52
+ "value": -3.0920298099517822
53
+ },
54
+ {
55
+ "decision_ref": "[0930-S2DEC]",
56
+ "key": "learner.qc.agent.actor_lr_partition",
57
+ "reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
58
+ "upstream": "lr=0.0003 shared by actor and critic",
59
+ "value": 0.0001
60
+ },
61
+ {
62
+ "decision_ref": "[1002-S2ALIGN]",
63
+ "key": "learner.qc.agent.actor_num_samples",
64
+ "reason": "single BoN knob including QC TD",
65
+ "upstream": 32,
66
+ "value": 16
67
+ },
68
+ {
69
+ "decision_ref": "[1002-S2ALIGN]",
70
+ "key": "learner.qc.agent.best_of_n",
71
+ "reason": "single BoN knob: collection/eval/QC TD",
72
+ "upstream": 8,
73
+ "value": 16
74
+ },
75
+ {
76
+ "decision_ref": "docs/scope.md [0928-I]",
77
+ "key": "learner.qc.agent.bon_agg",
78
+ "reason": "Best-of-N selection by the ensemble mean",
79
+ "upstream": null,
80
+ "value": "mean"
81
+ },
82
+ {
83
+ "decision_ref": "docs/scope.md [0928-I]",
84
+ "key": "learner.qc.agent.bon_sft",
85
+ "reason": "variant qc flag bon_sft",
86
+ "upstream": "upstream_yaml",
87
+ "value": false
88
+ },
89
+ {
90
+ "decision_ref": "[0930-S2DEC]",
91
+ "key": "learner.qc.agent.critic_lr_partition",
92
+ "reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
93
+ "upstream": "lr=0.0003 shared by actor and critic",
94
+ "value": 0.0003
95
+ },
96
+ {
97
+ "decision_ref": "docs/scope.md [0928-I]",
98
+ "key": "learner.qc.agent.partition_optimizer",
99
+ "reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
100
+ "upstream": null,
101
+ "value": true
102
+ },
103
+ {
104
+ "decision_ref": "docs/scope.md [0928-I]",
105
+ "key": "learner.qc.agent.q_target_agg",
106
+ "reason": "REDQ-style TD target: min over a random critic subset",
107
+ "upstream": null,
108
+ "value": "subset_min"
109
+ },
110
+ {
111
+ "decision_ref": "docs/scope.md [0928-I]",
112
+ "key": "learner.qc.agent.q_target_replace",
113
+ "reason": "subset drawn without replacement",
114
+ "upstream": null,
115
+ "value": false
116
+ },
117
+ {
118
+ "decision_ref": "docs/scope.md [0928-I]",
119
+ "key": "learner.qc.agent.q_target_subset",
120
+ "reason": "TD-target subset size",
121
+ "upstream": null,
122
+ "value": 2
123
+ },
124
+ {
125
+ "decision_ref": "[0930-S2DEC]",
126
+ "key": "learner.qc.agent.tau",
127
+ "reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
128
+ "upstream": 0.05,
129
+ "value": 0.005
130
+ },
131
+ {
132
+ "decision_ref": "[0930-S2DEC]",
133
+ "key": "learner.qc.agent.value_hidden_dims",
134
+ "reason": "unified.critic_hidden (critics only; actors native)",
135
+ "upstream": [
136
+ 512,
137
+ 512,
138
+ 512,
139
+ 512,
140
+ 512
141
+ ],
142
+ "value": [
143
+ 512,
144
+ 512,
145
+ 512
146
+ ]
147
+ },
148
+ {
149
+ "decision_ref": "docs/scope.md [0928-I]",
150
+ "key": "learner.qc.training.bc_pi_steps",
151
+ "reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
152
+ "upstream": 1000000,
153
+ "value": 0
154
+ },
155
+ {
156
+ "decision_ref": "docs/scope.md [0928-I]",
157
+ "key": "learner.qc.loop.buffer.utd_online",
158
+ "reason": "UTD=3 per executed action chunk",
159
+ "upstream": 1,
160
+ "value": "3 gradient updates per executed chunk"
161
+ },
162
+ {
163
+ "decision_ref": "docs/scope.md [0928-I]",
164
+ "key": "learner.qc.loop.n_step",
165
+ "reason": "n-step=3 counted in action chunks",
166
+ "upstream": "1 chunk (sequence_length = horizon_length)",
167
+ "value": "5 executed chunks (unified_env_step window)"
168
+ },
169
+ {
170
+ "decision_ref": "docs/scope.md [0928-I]",
171
+ "key": "learner.qc.loop.critic_actor_ratio",
172
+ "reason": "critic every update, actor every 2nd global update",
173
+ "upstream": "1:1 (utd_q = utd_pi)",
174
+ "value": "2:1"
175
+ },
176
+ {
177
+ "decision_ref": "docs/scope.md [0928-I]",
178
+ "key": "learner.qc.loop.buffer.start_training",
179
+ "reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
180
+ "upstream": 10000,
181
+ "value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
182
+ },
183
+ {
184
+ "decision_ref": "docs/scope.md [0928-I]",
185
+ "key": "learner.qc.loop.buffer.offline_ratio",
186
+ "reason": "[1017-S2OFFBC] adaptive_ratio off",
187
+ "upstream": 0.0,
188
+ "value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
189
+ },
190
+ {
191
+ "decision_ref": "docs/scope.md [0928-I]",
192
+ "key": "learner.qc.loop.buffer.size",
193
+ "reason": "one shared replay of executed chunks; success buffer = mask",
194
+ "upstream": 2000000,
195
+ "value": "1000000 executed chunks, shared FIFO"
196
+ },
197
+ {
198
+ "decision_ref": "docs/scope.md [0928-I]",
199
+ "key": "learner.qc.actor_architecture",
200
+ "reason": "Study2 fine-tunes the Study1 pixel_state CFM",
201
+ "upstream": [
202
+ 512,
203
+ 512,
204
+ 512,
205
+ 512,
206
+ 512
207
+ ],
208
+ "value": "checkpoint StandardGenerator (EMA generator copy)"
209
+ },
210
+ {
211
+ "decision_ref": "docs/scope.md [0928-I]",
212
+ "key": "learner.qc.observation",
213
+ "reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
214
+ "upstream": "low-dim state",
215
+ "value": "frozen Ri features c [137] from the checkpoint encoder"
216
+ },
217
+ {
218
+ "decision_ref": "[0930-S2DEC]",
219
+ "key": "learner.qc.reward",
220
+ "reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
221
+ "upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
222
+ "value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
223
+ },
224
+ {
225
+ "decision_ref": "[0930-S2DEC]",
226
+ "key": "learner.qc.td_target",
227
+ "reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
228
+ "upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
229
+ "value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
230
+ },
231
+ {
232
+ "decision_ref": "[0930-S2DEC]",
233
+ "key": "learner.qc.eval",
234
+ "reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
235
+ "upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
236
+ "value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
237
+ },
238
+ {
239
+ "decision_ref": "[1002-S2ALIGN]",
240
+ "key": "learner.qc.lr_schedule",
241
+ "reason": "RL env-step clock shared with extra success TD",
242
+ "upstream": "native per-optimizer schedule",
243
+ "value": {
244
+ "actor": 0.0001,
245
+ "critic": 0.0003,
246
+ "decay_end_env_steps": 1606196,
247
+ "min_ratio": 0.1,
248
+ "schedule": "warmup_cosine",
249
+ "temperature": 0.0003,
250
+ "warmup_env_steps": 20000
251
+ }
252
+ },
253
+ {
254
+ "decision_ref": "[0930-S2DEC]",
255
+ "key": "learner.qc.agent.q_min_q_max",
256
+ "reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
257
+ "upstream": [
258
+ -999.9999999999991,
259
+ 0.0
260
+ ],
261
+ "value": [
262
+ 0.0,
263
+ 1.0
264
+ ]
265
+ }
266
+ ],
267
+ "protocol": {
268
+ "checkpoint_every_rounds": 5,
269
+ "checkpoint_retention": {
270
+ "best": true,
271
+ "collapse_drop": {
272
+ "min_abs": 0.2,
273
+ "ratio": 0.5
274
+ },
275
+ "diagnostics": true,
276
+ "every_evals": 3
277
+ },
278
+ "collection": {
279
+ "n_envs": 4,
280
+ "train_seed_base": 100000000,
281
+ "train_seed_stride": 1000000,
282
+ "training_freq_chunks": 1000
283
+ },
284
+ "convergence": {
285
+ "early_stop": false,
286
+ "gain_lag": 10,
287
+ "high_sr": 0.95,
288
+ "high_sr_evals": 5,
289
+ "ma_window": 10,
290
+ "max_env_steps": 1606196,
291
+ "min_env_steps": 200000,
292
+ "min_gain": 0.02,
293
+ "zero_success_evals": 8
294
+ },
295
+ "critic_ensemble": {
296
+ "num": 10,
297
+ "policy": "mean",
298
+ "target": {
299
+ "kind": "subset_min",
300
+ "replace": false,
301
+ "size": 2
302
+ }
303
+ },
304
+ "eval": {
305
+ "best_of_n": 16,
306
+ "episodes": 200,
307
+ "every_env_steps": 20000,
308
+ "include_step0": true,
309
+ "seed_start": 960000,
310
+ "workers": 4
311
+ },
312
+ "final_eval": {
313
+ "episodes": 300,
314
+ "on_zero_stop": true,
315
+ "seed_start": 800000
316
+ },
317
+ "n_step_chunks": 5,
318
+ "offline": {
319
+ "adaptive_ratio": false,
320
+ "cohort_size": 100,
321
+ "episodes": 100,
322
+ "learners_without_offline": [
323
+ "dsrl_sac"
324
+ ],
325
+ "ratio": {
326
+ "decay_env_steps": {
327
+ "can": 160000,
328
+ "square": 320000,
329
+ "tool_hang": 640000,
330
+ "transport": 320000
331
+ },
332
+ "end": 0.1,
333
+ "start": {
334
+ "can": 0.7,
335
+ "square": 0.5,
336
+ "tool_hang": 0.5,
337
+ "transport": 0.9
338
+ }
339
+ },
340
+ "selection": "ascending_demo_index",
341
+ "source": "bc_regime",
342
+ "unsupported_learners": [
343
+ "zprl"
344
+ ]
345
+ },
346
+ "replay": {
347
+ "capacity_chunks": 1000000,
348
+ "pin_start_successes": false,
349
+ "success_buffer": "success_mask_on_shared_fifo",
350
+ "success_skip_rule": "skip_below_success_batch_size",
351
+ "unit": "executed_chunk"
352
+ },
353
+ "start": {
354
+ "max_episodes": 200,
355
+ "mode": "offline_buffer",
356
+ "policy": {
357
+ "default": "base_cfm_ode",
358
+ "zprl": "vib_base"
359
+ },
360
+ "target_successes": 20
361
+ },
362
+ "unified": {
363
+ "actor_target_tau": 0.005,
364
+ "critic_hidden": [
365
+ 512,
366
+ 512,
367
+ 512
368
+ ],
369
+ "critic_target_every_critic_step": true,
370
+ "critic_target_tau": 0.005,
371
+ "dsrl_noise_imitation_reference_dims": 28,
372
+ "gamma_env": 0.999,
373
+ "lr": {
374
+ "actor": 0.0001,
375
+ "critic": 0.0003,
376
+ "decay_end_env_steps": 1606196,
377
+ "min_ratio": 0.1,
378
+ "schedule": "warmup_cosine",
379
+ "temperature": 0.0003,
380
+ "warmup_env_steps": 20000
381
+ },
382
+ "reward": "binary_first_success",
383
+ "success_td_polyak": true,
384
+ "timeout_bootstrap": false,
385
+ "zprl_imitation": "mse_pretanh_mean"
386
+ },
387
+ "updates": {
388
+ "batch_size": 256,
389
+ "critic_per_actor": 2,
390
+ "success_batch_size": 256,
391
+ "utd_per_chunk": 3
392
+ },
393
+ "version": "study2_protocol_v10"
394
+ },
395
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
396
+ "provenance_extra": {
397
+ "agent_library": {
398
+ "framework": "jax",
399
+ "sources": {
400
+ "__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
401
+ "_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
402
+ "_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
403
+ "_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
404
+ "_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
405
+ "_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
406
+ "_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
407
+ "_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
408
+ "_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
409
+ "_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
410
+ "_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
411
+ "_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
412
+ "_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
413
+ "_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
414
+ "_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
415
+ "_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
416
+ "_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
417
+ "_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
418
+ "_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
419
+ "_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
420
+ "_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
421
+ "_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
422
+ "_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
423
+ "_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
424
+ "_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
425
+ "_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
426
+ "_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
427
+ "_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
428
+ "_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
429
+ "_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
430
+ "_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
431
+ "dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
432
+ "dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
433
+ "dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
434
+ "factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
435
+ "flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
436
+ "native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
437
+ "native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
438
+ "native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
439
+ "native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
440
+ "native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
441
+ "networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
442
+ "ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
443
+ "optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
444
+ "qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
445
+ "registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
446
+ "residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
447
+ "telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
448
+ "train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
449
+ "vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
450
+ "vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
451
+ "zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
452
+ },
453
+ "version": "study2_jax_agents_v2"
454
+ },
455
+ "base_framework": "jax",
456
+ "bc_identity": {
457
+ "architecture": "dice_vision_resnet_flow_unet_v1",
458
+ "family": "StandardBaseBC",
459
+ "framework": "jax"
460
+ },
461
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/base/mixed100/epoch_300.pkl",
462
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
463
+ "experiment": "tool_hang",
464
+ "frozen_hash": "848c66c08785dca4f652c3a7eb0bd2cb26761fbdf91fb1dbcc2cde5be0446931",
465
+ "learner": {
466
+ "action_bounds": {
467
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
468
+ "compat_act_max": 3.0794031620025635,
469
+ "compat_act_min": -3.0920298099517822,
470
+ "hi": [
471
+ 1.0,
472
+ 1.0,
473
+ 1.0,
474
+ 3.0794031620025635,
475
+ 1.9159542322158813,
476
+ 1.0,
477
+ 1.0
478
+ ],
479
+ "kind": "per_dimension",
480
+ "lo": [
481
+ -1.0,
482
+ -1.0,
483
+ -1.0,
484
+ -3.0920298099517822,
485
+ -2.0611963272094727,
486
+ -1.531522274017334,
487
+ -1.0
488
+ ],
489
+ "note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
490
+ },
491
+ "agent_config": {
492
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
493
+ "act_max": 3.0794031620025635,
494
+ "act_min": -3.0920298099517822,
495
+ "action_chunking": true,
496
+ "action_critic_hidden_dims": null,
497
+ "action_dim": 7,
498
+ "action_num_qs": 10,
499
+ "action_repeat": false,
500
+ "actor_backbone": "mlp",
501
+ "actor_decay_steps": 50000,
502
+ "actor_end_value": 2e-05,
503
+ "actor_hidden_dims": [
504
+ 512,
505
+ 512,
506
+ 512,
507
+ 512,
508
+ 512
509
+ ],
510
+ "actor_layer_norm": false,
511
+ "actor_lr": 0.0003,
512
+ "actor_lr_partition": 0.0001,
513
+ "actor_num_samples": 16,
514
+ "actor_obs": "state",
515
+ "actor_scheduler": "cosine",
516
+ "actor_tau": 0.005,
517
+ "actor_type": "best-of-n",
518
+ "actor_warmup_steps": 2000,
519
+ "actor_weight_decay": 0.0,
520
+ "adv_clip_min": null,
521
+ "agent_name": "acfql",
522
+ "alpha": 500,
523
+ "batch_size": 256,
524
+ "best_of_n": 16,
525
+ "bon_agg": "mean",
526
+ "bon_sft": false,
527
+ "bon_sft_coeff": 1.0,
528
+ "bootstrap_ratio": 0.5,
529
+ "bptt_num_samples": 4,
530
+ "calql_q_subsample": 2,
531
+ "clip_grad_norm": 1000.0,
532
+ "clip_intermediate_actions": true,
533
+ "clip_max_epsilon_multiplier": 1.0,
534
+ "clip_min_epsilon_multiplier": 1.0,
535
+ "constant_noise_std": 0.01,
536
+ "cql_alpha": 5.0,
537
+ "cql_n_actions": 10,
538
+ "cql_temp": 1.0,
539
+ "critic_backbone": "mlp",
540
+ "critic_decay_steps": 5000,
541
+ "critic_end_value": 1e-08,
542
+ "critic_loss_type": "mse",
543
+ "critic_lr": 0.0003,
544
+ "critic_lr_partition": 0.0003,
545
+ "critic_obs": "state",
546
+ "critic_scheduler": "constant",
547
+ "critic_warmup_steps": 500,
548
+ "critic_weight_decay": 1e-05,
549
+ "denoised_clip_value": 1.0,
550
+ "discount": 0.999,
551
+ "edit_action_scale": 0.5,
552
+ "edit_actor_hidden_dims": null,
553
+ "edit_hidden_dims": [
554
+ 256,
555
+ 256,
556
+ 256
557
+ ],
558
+ "edit_init_temperature": 1.0,
559
+ "edit_layer_norm": true,
560
+ "edit_lr": 0.0003,
561
+ "edit_policy_bound": 0.1,
562
+ "edit_start_step": 100000,
563
+ "edit_target_entropy": null,
564
+ "edit_weight_decay": 0.0,
565
+ "encoder": null,
566
+ "entropy_backup": false,
567
+ "flow_steps": 10,
568
+ "fql_distillation_coeff": 10.0,
569
+ "fql_train_main_policy": true,
570
+ "freeze_vision_encoder": false,
571
+ "ft_flow_steps": 10,
572
+ "horizon_length": 8,
573
+ "init_temperature": 1.0,
574
+ "layer_norm": true,
575
+ "lr": 0.0003,
576
+ "lr_action_critic": null,
577
+ "lr_actor": null,
578
+ "lr_actor_bc_flow": null,
579
+ "lr_critic": null,
580
+ "lr_edit_actor": null,
581
+ "lr_noise_critic": null,
582
+ "max_noise_std": 0.01,
583
+ "mc_regression": false,
584
+ "mc_regression_coeff": 1.0,
585
+ "min_noise_std": 0.01,
586
+ "mip_q_ensemble": false,
587
+ "mip_q_noise_scale": 1.0,
588
+ "mip_q_t_star": 0.9,
589
+ "mip_t_star": 0.999,
590
+ "muon_beta": 0.95,
591
+ "muon_nesterov": true,
592
+ "muon_ns_steps": 5,
593
+ "n_action_samples": 8,
594
+ "n_edit_samples": 8,
595
+ "noise_critic_grad_steps": 1,
596
+ "noise_dim": null,
597
+ "noise_net_hidden_dims": [
598
+ 256,
599
+ 256
600
+ ],
601
+ "noise_net_layer_norm": true,
602
+ "normalize_act_space_dimension": false,
603
+ "normalize_denoising_horizon": false,
604
+ "normalize_q_loss": false,
605
+ "num_bins": 256,
606
+ "num_ensemble_members": 10,
607
+ "num_qs": 10,
608
+ "ob_dims": [
609
+ 137
610
+ ],
611
+ "one_step_decay_steps": null,
612
+ "one_step_end_value": null,
613
+ "one_step_hidden_dims": [
614
+ 512,
615
+ 512,
616
+ 512,
617
+ 512
618
+ ],
619
+ "one_step_layer_norm": false,
620
+ "one_step_lr": null,
621
+ "one_step_scheduler": null,
622
+ "one_step_warmup_steps": null,
623
+ "optimizer_action_critic": null,
624
+ "optimizer_actor": "adam",
625
+ "optimizer_actor_bc_flow": "adam",
626
+ "optimizer_critic": "adam",
627
+ "optimizer_edit_actor": null,
628
+ "paligemma_tokenizer_path": null,
629
+ "partition_optimizer": true,
630
+ "policy_type": "flow",
631
+ "ppo_batch_size": 256,
632
+ "ppo_lr": 4.5e-05,
633
+ "proprio_dim": null,
634
+ "q_agg": "mean",
635
+ "q_max": 1.0,
636
+ "q_min": 0.0,
637
+ "q_target_agg": "subset_min",
638
+ "q_target_replace": false,
639
+ "q_target_subset": 2,
640
+ "q_variance_reduction": false,
641
+ "q_vr_num_samples": 8,
642
+ "q_vr_reduction": "mean",
643
+ "randn_clip_value": 3.0,
644
+ "shortcut_denoising_steps": 4,
645
+ "shortcut_inference_steps": 4,
646
+ "simba_actor_hidden_dims": [
647
+ 1024,
648
+ 1024
649
+ ],
650
+ "simba_critic_hidden_dims": [
651
+ 1024,
652
+ 1024,
653
+ 1024
654
+ ],
655
+ "subsample_bon": false,
656
+ "target_entropy": null,
657
+ "tau": 0.005,
658
+ "tf_pi_conditioning": "adaln",
659
+ "tf_pi_dropout": 0.0,
660
+ "tf_pi_embed_dim": 256,
661
+ "tf_pi_heads": 4,
662
+ "tf_pi_layers": 8,
663
+ "tf_q_conditioning": "adaln",
664
+ "tf_q_dropout": 0.0,
665
+ "tf_q_embed_dim": 256,
666
+ "tf_q_heads": 4,
667
+ "tf_q_layers": 8,
668
+ "time_embedding": "sinusoidal",
669
+ "time_embedding_dim": 32,
670
+ "train_one_step_in_offline": true,
671
+ "train_one_step_in_online": true,
672
+ "useFiLM": false,
673
+ "useSimBa": false,
674
+ "useTF": false,
675
+ "use_constant_noise": true,
676
+ "use_constant_scheduler_for_bc": true,
677
+ "use_denoiser": false,
678
+ "use_edit_actor": false,
679
+ "use_mip_q": false,
680
+ "use_muon": false,
681
+ "use_one_step_policy": false,
682
+ "use_shortcut": false,
683
+ "use_state": "proprio",
684
+ "use_success_buffer_q": false,
685
+ "value_hidden_dims": [
686
+ 512,
687
+ 512,
688
+ 512
689
+ ],
690
+ "vit_encoder": "minvit",
691
+ "vit_img_proj_dim": 0,
692
+ "vit_state_proj_dim": 0
693
+ },
694
+ "convention": "unified_env_step",
695
+ "eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
696
+ "gamma_env": 0.999,
697
+ "gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
698
+ "learner": "qc",
699
+ "optimizers": {
700
+ "actor_lr": 0.0001,
701
+ "critic_lr": 0.0003,
702
+ "reset_optimizers_with_lr": "not used",
703
+ "schedule": {
704
+ "actor": 0.0001,
705
+ "critic": 0.0003,
706
+ "decay_end_env_steps": 1606196,
707
+ "min_ratio": 0.1,
708
+ "schedule": "warmup_cosine",
709
+ "temperature": 0.0003,
710
+ "warmup_env_steps": 20000
711
+ }
712
+ },
713
+ "qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
714
+ "recipe": {
715
+ "derived": {
716
+ "agent.actor_num_samples": {
717
+ "after": 8,
718
+ "before": 32
719
+ },
720
+ "agent.actor_type": {
721
+ "after": "best-of-n",
722
+ "before": "upstream_yaml"
723
+ },
724
+ "agent.bon_sft": {
725
+ "after": false,
726
+ "before": "upstream_yaml"
727
+ }
728
+ },
729
+ "explicit_overrides": {},
730
+ "source": "third_party/ogpo/scripts/qc/toolhang.sh",
731
+ "upstream_exact": false
732
+ },
733
+ "reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
734
+ "rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
735
+ "scan_block": 100,
736
+ "start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
737
+ "success_terms": {
738
+ "bc_on_success_batch": false,
739
+ "critic_success_td": false,
740
+ "missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
741
+ },
742
+ "td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
743
+ "unified_notes": {
744
+ "critic_target_every_critic_step": {
745
+ "note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
746
+ "value": true
747
+ },
748
+ "success_td_polyak": {
749
+ "note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
750
+ "value": true
751
+ }
752
+ },
753
+ "upstream_config": {
754
+ "agent": {
755
+ "act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -3.0920298099517822, -2.0611963272094727, -1.531522274017334, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0794031620025635, 1.9159542322158813, 1.0, 1.0]}",
756
+ "act_max": 3.0794031620025635,
757
+ "act_min": -3.0920298099517822,
758
+ "action_chunking": true,
759
+ "action_critic_hidden_dims": null,
760
+ "action_dim": null,
761
+ "action_num_qs": 10,
762
+ "action_repeat": false,
763
+ "actor_backbone": "mlp",
764
+ "actor_decay_steps": 50000,
765
+ "actor_end_value": 2e-05,
766
+ "actor_hidden_dims": [
767
+ 512,
768
+ 512,
769
+ 512,
770
+ 512,
771
+ 512
772
+ ],
773
+ "actor_layer_norm": false,
774
+ "actor_lr": 0.0003,
775
+ "actor_lr_partition": 0.0001,
776
+ "actor_num_samples": 16,
777
+ "actor_obs": "state",
778
+ "actor_scheduler": "cosine",
779
+ "actor_tau": 0.005,
780
+ "actor_type": "best-of-n",
781
+ "actor_warmup_steps": 2000,
782
+ "actor_weight_decay": 0.0,
783
+ "adv_clip_min": null,
784
+ "agent_name": "acfql",
785
+ "alpha": 500,
786
+ "batch_size": 256,
787
+ "best_of_n": 16,
788
+ "bon_agg": "mean",
789
+ "bon_sft": false,
790
+ "bon_sft_coeff": 1.0,
791
+ "bootstrap_ratio": 0.5,
792
+ "bptt_num_samples": 4,
793
+ "calql_q_subsample": 2,
794
+ "clip_grad_norm": 1000.0,
795
+ "clip_intermediate_actions": true,
796
+ "clip_max_epsilon_multiplier": 1.0,
797
+ "clip_min_epsilon_multiplier": 1.0,
798
+ "constant_noise_std": 0.01,
799
+ "cql_alpha": 5.0,
800
+ "cql_n_actions": 10,
801
+ "cql_temp": 1.0,
802
+ "critic_backbone": "mlp",
803
+ "critic_decay_steps": 5000,
804
+ "critic_end_value": 1e-08,
805
+ "critic_loss_type": "mse",
806
+ "critic_lr": 0.0003,
807
+ "critic_lr_partition": 0.0003,
808
+ "critic_obs": "state",
809
+ "critic_scheduler": "constant",
810
+ "critic_warmup_steps": 500,
811
+ "critic_weight_decay": 1e-05,
812
+ "denoised_clip_value": 1.0,
813
+ "discount": 0.999,
814
+ "edit_action_scale": 0.5,
815
+ "edit_actor_hidden_dims": null,
816
+ "edit_hidden_dims": [
817
+ 256,
818
+ 256,
819
+ 256
820
+ ],
821
+ "edit_init_temperature": 1.0,
822
+ "edit_layer_norm": true,
823
+ "edit_lr": 0.0003,
824
+ "edit_policy_bound": 0.1,
825
+ "edit_start_step": 100000,
826
+ "edit_target_entropy": null,
827
+ "edit_weight_decay": 0.0,
828
+ "encoder": null,
829
+ "entropy_backup": false,
830
+ "flow_steps": 10,
831
+ "fql_distillation_coeff": 10.0,
832
+ "fql_train_main_policy": true,
833
+ "freeze_vision_encoder": false,
834
+ "ft_flow_steps": 10,
835
+ "horizon_length": 8,
836
+ "init_temperature": 1.0,
837
+ "layer_norm": true,
838
+ "lr": 0.0003,
839
+ "lr_action_critic": null,
840
+ "lr_actor": null,
841
+ "lr_actor_bc_flow": null,
842
+ "lr_critic": null,
843
+ "lr_edit_actor": null,
844
+ "lr_noise_critic": null,
845
+ "max_noise_std": 0.01,
846
+ "mc_regression": false,
847
+ "mc_regression_coeff": 1.0,
848
+ "min_noise_std": 0.01,
849
+ "mip_q_ensemble": false,
850
+ "mip_q_noise_scale": 1.0,
851
+ "mip_q_t_star": 0.9,
852
+ "mip_t_star": 0.999,
853
+ "muon_beta": 0.95,
854
+ "muon_nesterov": true,
855
+ "muon_ns_steps": 5,
856
+ "n_action_samples": 8,
857
+ "n_edit_samples": 8,
858
+ "noise_critic_grad_steps": 1,
859
+ "noise_dim": null,
860
+ "noise_net_hidden_dims": [
861
+ 256,
862
+ 256
863
+ ],
864
+ "noise_net_layer_norm": true,
865
+ "normalize_act_space_dimension": false,
866
+ "normalize_denoising_horizon": false,
867
+ "normalize_q_loss": false,
868
+ "num_bins": 256,
869
+ "num_ensemble_members": 10,
870
+ "num_qs": 10,
871
+ "ob_dims": null,
872
+ "one_step_decay_steps": null,
873
+ "one_step_end_value": null,
874
+ "one_step_hidden_dims": [
875
+ 512,
876
+ 512,
877
+ 512,
878
+ 512
879
+ ],
880
+ "one_step_layer_norm": false,
881
+ "one_step_lr": null,
882
+ "one_step_scheduler": null,
883
+ "one_step_warmup_steps": null,
884
+ "optimizer_action_critic": null,
885
+ "optimizer_actor": "adam",
886
+ "optimizer_actor_bc_flow": "adam",
887
+ "optimizer_critic": "adam",
888
+ "optimizer_edit_actor": null,
889
+ "paligemma_tokenizer_path": null,
890
+ "partition_optimizer": true,
891
+ "policy_type": "flow",
892
+ "ppo_batch_size": 256,
893
+ "ppo_lr": 4.5e-05,
894
+ "proprio_dim": null,
895
+ "q_agg": "mean",
896
+ "q_max": null,
897
+ "q_min": null,
898
+ "q_target_agg": "subset_min",
899
+ "q_target_replace": false,
900
+ "q_target_subset": 2,
901
+ "q_variance_reduction": false,
902
+ "q_vr_num_samples": 8,
903
+ "q_vr_reduction": "mean",
904
+ "randn_clip_value": 3.0,
905
+ "shortcut_denoising_steps": 4,
906
+ "shortcut_inference_steps": 4,
907
+ "simba_actor_hidden_dims": [
908
+ 1024,
909
+ 1024
910
+ ],
911
+ "simba_critic_hidden_dims": [
912
+ 1024,
913
+ 1024,
914
+ 1024
915
+ ],
916
+ "subsample_bon": false,
917
+ "target_entropy": null,
918
+ "tau": 0.005,
919
+ "tf_pi_conditioning": "adaln",
920
+ "tf_pi_dropout": 0.0,
921
+ "tf_pi_embed_dim": 256,
922
+ "tf_pi_heads": 4,
923
+ "tf_pi_layers": 8,
924
+ "tf_q_conditioning": "adaln",
925
+ "tf_q_dropout": 0.0,
926
+ "tf_q_embed_dim": 256,
927
+ "tf_q_heads": 4,
928
+ "tf_q_layers": 8,
929
+ "time_embedding": "sinusoidal",
930
+ "time_embedding_dim": 32,
931
+ "train_one_step_in_offline": true,
932
+ "train_one_step_in_online": true,
933
+ "useFiLM": false,
934
+ "useSimBa": false,
935
+ "useTF": false,
936
+ "use_constant_noise": true,
937
+ "use_constant_scheduler_for_bc": true,
938
+ "use_denoiser": false,
939
+ "use_edit_actor": false,
940
+ "use_mip_q": false,
941
+ "use_muon": false,
942
+ "use_one_step_policy": false,
943
+ "use_shortcut": false,
944
+ "use_state": "proprio",
945
+ "use_success_buffer_q": false,
946
+ "value_hidden_dims": [
947
+ 512,
948
+ 512,
949
+ 512
950
+ ],
951
+ "vit_encoder": "minvit",
952
+ "vit_img_proj_dim": 0,
953
+ "vit_state_proj_dim": 0
954
+ },
955
+ "bc": {
956
+ "clip_bc": true,
957
+ "clip_bc_threshold": 0.45,
958
+ "clip_bc_wrt": "ODE"
959
+ },
960
+ "buffer": {
961
+ "offline_ratio": 0.0,
962
+ "size": 2000000,
963
+ "start_training": 10000,
964
+ "utd_online": 1,
965
+ "utd_pi": null,
966
+ "utd_q": null,
967
+ "utd_v": null,
968
+ "utd_warmup": 1
969
+ },
970
+ "checkpoint": {
971
+ "ep_resume": 0,
972
+ "restore_actor_path": null,
973
+ "restore_critic_path": null,
974
+ "restore_encoder_path": null
975
+ },
976
+ "dataset": {
977
+ "num_offline_trajs": -1,
978
+ "ogbench_dir": null,
979
+ "p_aug": 0.0,
980
+ "proportion": 1.0,
981
+ "replace_interval": 1000
982
+ },
983
+ "env": {
984
+ "name": "tool_hang-ph-low_dim",
985
+ "post_success_steps": 0,
986
+ "rew_fn": "sparse",
987
+ "sparse": false
988
+ },
989
+ "eval": {
990
+ "eval_episodes": 2,
991
+ "n_eval_envs": 32,
992
+ "plot_q_vs_mc": false,
993
+ "video_episodes": 0,
994
+ "video_frame_skip": 3
995
+ },
996
+ "experiment": {
997
+ "log": true,
998
+ "project": "QC",
999
+ "run_group": "toolhang",
1000
+ "run_name": "qc_toolhang",
1001
+ "save_dir": "exp/ogpo",
1002
+ "seed": 1,
1003
+ "wandb_name": "qc_toolhang"
1004
+ },
1005
+ "intervals": {
1006
+ "eval_interval": 20000,
1007
+ "eval_interval_bc": 5000,
1008
+ "log_interval": 5000,
1009
+ "preemption_backup_interval": -1,
1010
+ "save_interval": 500000,
1011
+ "save_interval_bc": -1,
1012
+ "step_data_log_interval": -1
1013
+ },
1014
+ "success_buffer": {
1015
+ "batch_size": 256,
1016
+ "use_success_buffer": false
1017
+ },
1018
+ "training": {
1019
+ "bc_only": false,
1020
+ "bc_pi_steps": 0,
1021
+ "bc_q_steps": 0,
1022
+ "bc_refine_steps": 0,
1023
+ "calql_steps": 0,
1024
+ "force_jax_sync": true,
1025
+ "gpu_mode": "single",
1026
+ "online_steps": 5000000,
1027
+ "q_warmup_steps": 0
1028
+ }
1029
+ },
1030
+ "upstream_source": {
1031
+ "ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
1032
+ "patches_sha256": {
1033
+ "ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
1034
+ "ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
1035
+ "ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
1036
+ "ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
1037
+ "ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
1038
+ }
1039
+ }
1040
+ },
1041
+ "observation_mode": "pixel_state",
1042
+ "offline_buffer": {
1043
+ "actions_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1044
+ "aux_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
1045
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
1046
+ "chunks": 5946,
1047
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1048
+ "decision_ref": "[1017-S2OFFBC]",
1049
+ "env_steps": 47168,
1050
+ "features_sha256": "7af4a3aee67b4a7eae0c45d07974462977e71712cd76bb9afd89dfdb9130699b",
1051
+ "horizon": 8,
1052
+ "identity_hash": "d54f7fa39190225c48c52a3782ea8d00b6d59a0a08d5c6b085b1a6af164eff34",
1053
+ "learner_coordinates": "JaxStudy2Learner:qc",
1054
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1055
+ "ratio": {
1056
+ "adaptive_ratio": false,
1057
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
1058
+ },
1059
+ "selection": {
1060
+ "cohort_size": 100,
1061
+ "demos": [
1062
+ "demo_0",
1063
+ "demo_5",
1064
+ "demo_6",
1065
+ "demo_7",
1066
+ "demo_9",
1067
+ "demo_12",
1068
+ "demo_16",
1069
+ "demo_17",
1070
+ "demo_18",
1071
+ "demo_20",
1072
+ "demo_25",
1073
+ "demo_27",
1074
+ "demo_30",
1075
+ "demo_31",
1076
+ "demo_32",
1077
+ "demo_35",
1078
+ "demo_37",
1079
+ "demo_38",
1080
+ "demo_43",
1081
+ "demo_44",
1082
+ "demo_45",
1083
+ "demo_46",
1084
+ "demo_50",
1085
+ "demo_51",
1086
+ "demo_55",
1087
+ "demo_56",
1088
+ "demo_57",
1089
+ "demo_58",
1090
+ "demo_69",
1091
+ "demo_70",
1092
+ "demo_71",
1093
+ "demo_72",
1094
+ "demo_74",
1095
+ "demo_78",
1096
+ "demo_79",
1097
+ "demo_81",
1098
+ "demo_82",
1099
+ "demo_84",
1100
+ "demo_85",
1101
+ "demo_86",
1102
+ "demo_90",
1103
+ "demo_91",
1104
+ "demo_92",
1105
+ "demo_95",
1106
+ "demo_96",
1107
+ "demo_97",
1108
+ "demo_98",
1109
+ "demo_101",
1110
+ "demo_104",
1111
+ "demo_107",
1112
+ "demo_111",
1113
+ "demo_112",
1114
+ "demo_113",
1115
+ "demo_115",
1116
+ "demo_116",
1117
+ "demo_119",
1118
+ "demo_121",
1119
+ "demo_123",
1120
+ "demo_125",
1121
+ "demo_127",
1122
+ "demo_128",
1123
+ "demo_131",
1124
+ "demo_132",
1125
+ "demo_133",
1126
+ "demo_134",
1127
+ "demo_137",
1128
+ "demo_138",
1129
+ "demo_140",
1130
+ "demo_141",
1131
+ "demo_142",
1132
+ "demo_145",
1133
+ "demo_149",
1134
+ "demo_150",
1135
+ "demo_151",
1136
+ "demo_154",
1137
+ "demo_157",
1138
+ "demo_159",
1139
+ "demo_161",
1140
+ "demo_164",
1141
+ "demo_165",
1142
+ "demo_169",
1143
+ "demo_171",
1144
+ "demo_172",
1145
+ "demo_173",
1146
+ "demo_175",
1147
+ "demo_176",
1148
+ "demo_177",
1149
+ "demo_178",
1150
+ "demo_179",
1151
+ "demo_181",
1152
+ "demo_182",
1153
+ "demo_183",
1154
+ "demo_184",
1155
+ "demo_185",
1156
+ "demo_187",
1157
+ "demo_188",
1158
+ "demo_191",
1159
+ "demo_192",
1160
+ "demo_193",
1161
+ "demo_196"
1162
+ ],
1163
+ "manifest": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/source_input/data_manifest.json",
1164
+ "manifest_sha256": "6bcfaec343fc97c766bacf650dc6a73a71c0c9969974d21bfbfd1c1ac113685f",
1165
+ "regime": "mixed100",
1166
+ "source": "bc_regime",
1167
+ "source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
1168
+ "split": {
1169
+ "training": 90,
1170
+ "validation": 10
1171
+ },
1172
+ "task": "tool_hang"
1173
+ }
1174
+ },
1175
+ "offline_role": "mixed",
1176
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
1177
+ "smoke": false,
1178
+ "task": "tool_hang",
1179
+ "task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
1180
+ },
1181
+ "seed": 1,
1182
+ "start_policy": "base_cfm_ode",
1183
+ "variant": {
1184
+ "flags": {
1185
+ "bon_sft": false,
1186
+ "use_success_buffer": false
1187
+ },
1188
+ "learner": "qc",
1189
+ "name": "qc",
1190
+ "phase": 1,
1191
+ "spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
1192
+ "version": "study2_variants_v4"
1193
+ },
1194
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
1195
+ }
RL/ToolHang100/QC/mixed100/seed_1/start_protocol.json ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "committed_episodes": [],
3
+ "mode": "offline_buffer",
4
+ "note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
5
+ "offline": {
6
+ "adaptive_ratio": false,
7
+ "batches": 0,
8
+ "decay_env_steps": null,
9
+ "identity_hash": "4996633f55a4706fd52ff99ff6c3b390bcf374a5b7e4be59e0f6b6b77a77040e",
10
+ "last_ratio": null,
11
+ "offline_rows": 0,
12
+ "online_rows": 0,
13
+ "ratio_end": null,
14
+ "ratio_start": null,
15
+ "records": 5946
16
+ },
17
+ "offline_identity": {
18
+ "actions_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
19
+ "aux_sha256": "2f954a915ec80cd4c23577e7c4f901c04ed8d6adf16fb8bcf0a6f075318263c0",
20
+ "checkpoint_sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8",
21
+ "chunks": 5946,
22
+ "dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
23
+ "decision_ref": "[1017-S2OFFBC]",
24
+ "env_steps": 47168,
25
+ "features_sha256": "7af4a3aee67b4a7eae0c45d07974462977e71712cd76bb9afd89dfdb9130699b",
26
+ "horizon": 8,
27
+ "learner_coordinates": "JaxStudy2Learner:qc",
28
+ "pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
29
+ "ratio": {
30
+ "adaptive_ratio": false,
31
+ "rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
32
+ },
33
+ "selection": {
34
+ "cohort_size": 100,
35
+ "demos": [
36
+ "demo_0",
37
+ "demo_5",
38
+ "demo_6",
39
+ "demo_7",
40
+ "demo_9",
41
+ "demo_12",
42
+ "demo_16",
43
+ "demo_17",
44
+ "demo_18",
45
+ "demo_20",
46
+ "demo_25",
47
+ "demo_27",
48
+ "demo_30",
49
+ "demo_31",
50
+ "demo_32",
51
+ "demo_35",
52
+ "demo_37",
53
+ "demo_38",
54
+ "demo_43",
55
+ "demo_44",
56
+ "demo_45",
57
+ "demo_46",
58
+ "demo_50",
59
+ "demo_51",
60
+ "demo_55",
61
+ "demo_56",
62
+ "demo_57",
63
+ "demo_58",
64
+ "demo_69",
65
+ "demo_70",
66
+ "demo_71",
67
+ "demo_72",
68
+ "demo_74",
69
+ "demo_78",
70
+ "demo_79",
71
+ "demo_81",
72
+ "demo_82",
73
+ "demo_84",
74
+ "demo_85",
75
+ "demo_86",
76
+ "demo_90",
77
+ "demo_91",
78
+ "demo_92",
79
+ "demo_95",
80
+ "demo_96",
81
+ "demo_97",
82
+ "demo_98",
83
+ "demo_101",
84
+ "demo_104",
85
+ "demo_107",
86
+ "demo_111",
87
+ "demo_112",
88
+ "demo_113",
89
+ "demo_115",
90
+ "demo_116",
91
+ "demo_119",
92
+ "demo_121",
93
+ "demo_123",
94
+ "demo_125",
95
+ "demo_127",
96
+ "demo_128",
97
+ "demo_131",
98
+ "demo_132",
99
+ "demo_133",
100
+ "demo_134",
101
+ "demo_137",
102
+ "demo_138",
103
+ "demo_140",
104
+ "demo_141",
105
+ "demo_142",
106
+ "demo_145",
107
+ "demo_149",
108
+ "demo_150",
109
+ "demo_151",
110
+ "demo_154",
111
+ "demo_157",
112
+ "demo_159",
113
+ "demo_161",
114
+ "demo_164",
115
+ "demo_165",
116
+ "demo_169",
117
+ "demo_171",
118
+ "demo_172",
119
+ "demo_173",
120
+ "demo_175",
121
+ "demo_176",
122
+ "demo_177",
123
+ "demo_178",
124
+ "demo_179",
125
+ "demo_181",
126
+ "demo_182",
127
+ "demo_183",
128
+ "demo_184",
129
+ "demo_185",
130
+ "demo_187",
131
+ "demo_188",
132
+ "demo_191",
133
+ "demo_192",
134
+ "demo_193",
135
+ "demo_196"
136
+ ],
137
+ "manifest": "/home/nhy/RL-postTrain/repo/runs/operations/20261024_toolhang_mixed100_qc_v10/source_input/data_manifest.json",
138
+ "manifest_sha256": "6bcfaec343fc97c766bacf650dc6a73a71c0c9969974d21bfbfd1c1ac113685f",
139
+ "regime": "mixed100",
140
+ "source": "bc_regime",
141
+ "source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
142
+ "split": {
143
+ "training": 90,
144
+ "validation": 10
145
+ },
146
+ "task": "tool_hang"
147
+ }
148
+ },
149
+ "policy": "base_cfm_ode",
150
+ "start_chunks": 0,
151
+ "start_env_steps": 0,
152
+ "status": "complete"
153
+ }
RL/ToolHang100/QC/mixed100/seed_1/status.json ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accounting": {
3
+ "evaluation": {
4
+ "calls": 82,
5
+ "episodes": 16500
6
+ },
7
+ "history_complete": true,
8
+ "normalized": {
9
+ "checkpoint_seconds_per_env_step": 0.00010203608728314193,
10
+ "chunks_per_10000_env_steps": 1251.1523362001944,
11
+ "denominator_env_steps": 1606519,
12
+ "diagnostics_seconds_per_env_step": 0.0002092278530830836,
13
+ "episodes_per_10000_env_steps": 11.303943495221656,
14
+ "evaluation_seconds_per_env_step": 0.049586412837924726,
15
+ "history_complete": true,
16
+ "projected_chunks": 200959.58777954074,
17
+ "projected_episodes": 1815.6348826251044,
18
+ "projected_training_seconds": 28188.89245389504,
19
+ "projected_wall_seconds": 108493.617793721,
20
+ "projection_budget_env_steps": 1606196,
21
+ "projection_is_estimate": true,
22
+ "rollout_seconds_per_env_step": 0.005473975266918125,
23
+ "training_seconds_per_env_step": 0.017550095040639525,
24
+ "updates_seconds_per_env_step": 0.012076119773721401,
25
+ "wall_seconds_per_env_step": 0.06754693561291461
26
+ },
27
+ "rollout": {
28
+ "chunks": 201000,
29
+ "env_steps": 1606519,
30
+ "episodes": 1816,
31
+ "start_chunks": 0,
32
+ "start_env_steps": 0,
33
+ "successes": 413,
34
+ "truncated_episodes": 1403
35
+ },
36
+ "schema_version": 1,
37
+ "sequence": 1380,
38
+ "sessions": 4,
39
+ "time": {
40
+ "checkpoint_seconds": 163.92291290602589,
41
+ "diagnostics_seconds": 336.1285213071824,
42
+ "evaluation_seconds": 79661.51436597,
43
+ "preparation_seconds": 139.86004593399412,
44
+ "restore_seconds": 17.303129891995923,
45
+ "rollout_seconds": 8794.045271834038,
46
+ "start_rollout_seconds": 0.0,
47
+ "training_seconds": 28194.56113459317,
48
+ "updates_seconds": 19400.51586275913,
49
+ "wall_hours": 30.14317651497888,
50
+ "wall_seconds": 108515.43545392397
51
+ }
52
+ },
53
+ "detail": null,
54
+ "evals": 81,
55
+ "final_eval": {
56
+ "episodes": 300,
57
+ "imputed": false,
58
+ "n_success": 70,
59
+ "observed": true,
60
+ "seed_start": 800000,
61
+ "sr": 0.23333333333333334
62
+ },
63
+ "learner": "qc",
64
+ "learner_counters": {
65
+ "actor_updates": 301500,
66
+ "actor_updates_applied": 301500,
67
+ "actor_updates_skipped": 0,
68
+ "clip_stats": {
69
+ "clip_mag_sum": 1795.747267127037,
70
+ "coords_clipped_frac": 0.033546286425017766,
71
+ "max_clip_mag": 0.22043204307556152,
72
+ "mean_clip_mag": 0.004755724402278188,
73
+ "n_clipped": 377597,
74
+ "n_coords": 11256000
75
+ },
76
+ "clip_stats_start": {
77
+ "clip_mag_sum": 0.0,
78
+ "coords_clipped_frac": 0.0,
79
+ "max_clip_mag": 0.0,
80
+ "mean_clip_mag": 0.0,
81
+ "n_clipped": 0,
82
+ "n_coords": 0
83
+ },
84
+ "critic_updates": 603000,
85
+ "policy_version": 201,
86
+ "success_missing": 0
87
+ },
88
+ "phase": 1,
89
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
90
+ "resumes": [
91
+ {
92
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoint/state.pkl",
93
+ "mode": "env_restart_truncated",
94
+ "rl_env_steps": 87926,
95
+ "round": 11,
96
+ "superseded": {
97
+ "eval_files": [],
98
+ "index": 1,
99
+ "jsonl": {},
100
+ "rl_env_steps": 87926
101
+ }
102
+ },
103
+ {
104
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoint/state.pkl",
105
+ "mode": "env_restart_truncated",
106
+ "rl_env_steps": 127899,
107
+ "round": 16,
108
+ "superseded": {
109
+ "eval_files": [],
110
+ "index": 2,
111
+ "jsonl": {},
112
+ "rl_env_steps": 127899
113
+ }
114
+ },
115
+ {
116
+ "checkpoint": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoint/state.pkl",
117
+ "mode": "env_restart_truncated",
118
+ "rl_env_steps": 1166947,
119
+ "round": 146,
120
+ "superseded": {
121
+ "eval_files": [],
122
+ "index": 3,
123
+ "jsonl": {},
124
+ "rl_env_steps": 1166947
125
+ }
126
+ }
127
+ ],
128
+ "schedule": {
129
+ "actor_updates": 301500,
130
+ "chunk_transitions": 201000,
131
+ "critic_updates": 603000,
132
+ "gradient_updates": 603000,
133
+ "rl_chunks": 201000,
134
+ "rl_env_steps": 1606519,
135
+ "round_index": 201,
136
+ "update_index": 603000
137
+ },
138
+ "seed": 1,
139
+ "start": {
140
+ "mode": "offline_buffer",
141
+ "note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
142
+ "offline": {
143
+ "adaptive_ratio": false,
144
+ "batches": 603000,
145
+ "decay_env_steps": null,
146
+ "identity_hash": "d54f7fa39190225c48c52a3782ea8d00b6d59a0a08d5c6b085b1a6af164eff34",
147
+ "last_ratio": 0.02873491006446749,
148
+ "offline_rows": 15855424,
149
+ "online_rows": 138512576,
150
+ "ratio_end": null,
151
+ "ratio_start": null,
152
+ "records": 5946
153
+ },
154
+ "start_chunks": 0,
155
+ "start_env_steps": 0,
156
+ "status": "complete"
157
+ },
158
+ "status": "budget_complete",
159
+ "stop": {
160
+ "eval_index": 80,
161
+ "gain": -0.0010000000000000009,
162
+ "ma": 0.22849999999999998,
163
+ "rl_env_steps": 1606519,
164
+ "sr": 0.19,
165
+ "status": "budget_complete",
166
+ "trend": "flat"
167
+ },
168
+ "success_sampling": {
169
+ "imitation": {
170
+ "requested": 0,
171
+ "rounds_with_skips": 0,
172
+ "sampled": 0,
173
+ "skipped": 0
174
+ },
175
+ "last_round": {
176
+ "imitation": {
177
+ "requested": 0,
178
+ "sampled": 0,
179
+ "skipped": 0
180
+ },
181
+ "requested": 0,
182
+ "round": 200,
183
+ "sampled": 0,
184
+ "skipped": 0
185
+ },
186
+ "requested": 0,
187
+ "rounds_with_skips": 0,
188
+ "sampled": 0,
189
+ "skipped": 0,
190
+ "td": {
191
+ "requested": 0,
192
+ "rounds_with_skips": 0,
193
+ "sampled": 0,
194
+ "skipped": 0
195
+ }
196
+ },
197
+ "variant": "qc",
198
+ "variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
199
+ "wandb": {
200
+ "enabled": false,
201
+ "error": null,
202
+ "mode": "disabled",
203
+ "project": null,
204
+ "reason": "RLPT_STUDY2_WANDB not set (bridge `run` sets it for formal jobs)",
205
+ "run_id": null,
206
+ "transport": null,
207
+ "url": null
208
+ }
209
+ }
RL/ToolHang100/QC/mixed100/seed_1/training_accounting.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "evaluation": {
3
+ "calls": 82,
4
+ "episodes": 16500
5
+ },
6
+ "history_complete": true,
7
+ "normalized": {
8
+ "checkpoint_seconds_per_env_step": 0.00010203608728314193,
9
+ "chunks_per_10000_env_steps": 1251.1523362001944,
10
+ "denominator_env_steps": 1606519,
11
+ "diagnostics_seconds_per_env_step": 0.0002092278530830836,
12
+ "episodes_per_10000_env_steps": 11.303943495221656,
13
+ "evaluation_seconds_per_env_step": 0.049586412837924726,
14
+ "history_complete": true,
15
+ "projected_chunks": 200959.58777954074,
16
+ "projected_episodes": 1815.6348826251044,
17
+ "projected_training_seconds": 28188.89245389504,
18
+ "projected_wall_seconds": 108493.617793721,
19
+ "projection_budget_env_steps": 1606196,
20
+ "projection_is_estimate": true,
21
+ "rollout_seconds_per_env_step": 0.005473975266918125,
22
+ "training_seconds_per_env_step": 0.017550095040639525,
23
+ "updates_seconds_per_env_step": 0.012076119773721401,
24
+ "wall_seconds_per_env_step": 0.06754693561291461
25
+ },
26
+ "rollout": {
27
+ "chunks": 201000,
28
+ "env_steps": 1606519,
29
+ "episodes": 1816,
30
+ "start_chunks": 0,
31
+ "start_env_steps": 0,
32
+ "successes": 413,
33
+ "truncated_episodes": 1403
34
+ },
35
+ "schema_version": 1,
36
+ "sequence": 1381,
37
+ "sessions": 4,
38
+ "time": {
39
+ "checkpoint_seconds": 163.92291290602589,
40
+ "diagnostics_seconds": 336.1285213071824,
41
+ "evaluation_seconds": 79661.51436597,
42
+ "preparation_seconds": 139.86004593399412,
43
+ "restore_seconds": 17.303129891995923,
44
+ "rollout_seconds": 8794.045271834038,
45
+ "start_rollout_seconds": 0.0,
46
+ "training_seconds": 28194.56113459317,
47
+ "updates_seconds": 19400.51586275913,
48
+ "wall_hours": 30.14317651497888,
49
+ "wall_seconds": 108515.43545392397
50
+ }
51
+ }
RL/ToolHang100/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ToolHang100 RL checkpoints
2
+
3
+ 1 completed Study2 phase-1 v10 run, seed 1: Mixed100 QC (2 final/best checkpoint files).
4
+
5
+ Paths: `RL/ToolHang100/<Algorithm>/<cohort>/seed_1/{final,best}/state.pkl`.
6
+
7
+ Each run includes original resolved configurations, terminal status, source checkpoint metadata, convergence history, the real 300-episode final evaluation, publication provenance and SHA256SUMS.
8
+
9
+ | Algorithm | Cohort | Final SR (300 episodes) | Best periodic SR (200 episodes) | Files |
10
+ |---|---|---:|---:|---|
11
+ | QC | mixed100 | 70/300 (23.3%) | 66/200 (33.0%) | [final](QC/mixed100/seed_1/final/state.pkl) / [best](QC/mixed100/seed_1/best/state.pkl) |
12
+
13
+ Training completed the 1,606,196-step budget at 1,606,519 environment steps. The best checkpoint was selected at 143,879 environment steps.
14
+
15
+ Best values are the maximum observed periodic measurements used for checkpoint selection; they are not independent final evaluations. Final and best therefore use different evaluation panels.
16
+
17
+ `state.pkl` contains the saved learner state (including model/optimizer/target/RNG state). This is a checkpoint-weight publication, not a complete training-run archive: replay segments and `offline_buffer.pkl` remain in the original source run. Resume workflows must also supply the matching replay/offline data and the original environment/source configuration.
18
+
19
+ The frozen BC dependency remains at `Robomimic/study1/ToolHang/mixed100/seed_1/checkpoints/epoch_300.pkl`; its pinned revision and SHA256 are recorded in `publication.json` and [completed_rl.json](completed_rl.json). Original resolved configurations retain source-machine paths for provenance; use the manifest to resolve published artifacts on another host.
20
+
21
+ SHA256SUMS in the run folder covers all published files within that folder. Existing BC and RL artifacts in this repository were preserved.
RL/ToolHang100/completed_rl.json ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "repo_id": "yangfy0627/RL-postTrain",
4
+ "task": "ToolHang100",
5
+ "phase": 1,
6
+ "protocol": "study2_protocol_v10",
7
+ "seed": 1,
8
+ "run_count": 1,
9
+ "checkpoint_count": 2,
10
+ "excluded_runs": [],
11
+ "selection": {
12
+ "final": "terminal learner after budget completion; observed 300-episode evaluation",
13
+ "best": "maximum observed periodic success rate on 200 episodes, earliest tie"
14
+ },
15
+ "base_bc_revision": "95320f4b68a221fc235e2897a21558fa43545015",
16
+ "runs": [
17
+ {
18
+ "task": "ToolHang100",
19
+ "environment_task": "tool_hang",
20
+ "algorithm": "QC",
21
+ "variant": "qc",
22
+ "cohort": "mixed100",
23
+ "seed": 1,
24
+ "phase": 1,
25
+ "protocol_version": "study2_protocol_v10",
26
+ "protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
27
+ "status": "budget_complete",
28
+ "source_host": "HRG2",
29
+ "source_run": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1",
30
+ "path": "RL/ToolHang100/QC/mixed100/seed_1",
31
+ "checkpoints": {
32
+ "final": {
33
+ "path": "RL/ToolHang100/QC/mixed100/seed_1/final/state.pkl",
34
+ "sha256": "70dda4b58013596119a7b3bbb3d85a2f871fba438451a99f2754c18290928ead",
35
+ "bytes": 207224453,
36
+ "source_path": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoints/final/session_1792988385165681572/checkpoint/state.pkl",
37
+ "evaluation": {
38
+ "episodes": 300,
39
+ "imputed": false,
40
+ "n_success": 70,
41
+ "observed": true,
42
+ "seed_start": 800000,
43
+ "sr": 0.23333333333333334
44
+ }
45
+ },
46
+ "best": {
47
+ "path": "RL/ToolHang100/QC/mixed100/seed_1/best/state.pkl",
48
+ "sha256": "6ddafc0ff69f6fdea7c2951cad4b6ba09f3e32375fd411f9cd7abbec81adb8d9",
49
+ "bytes": 207205195,
50
+ "source_path": "/home/nhy/RL-postTrain/repo/runs/experiments/study2/pixel_state/20261024_toolhang_mixed100_qc_v10_seed1/checkpoints/best/eval_00007_step_143879_session_1792903293409232482/checkpoint/state.pkl",
51
+ "evaluation": {
52
+ "chunk_transitions": 18000,
53
+ "episodes": 200,
54
+ "eval_index": 7,
55
+ "gain": null,
56
+ "gradient_updates": 54000,
57
+ "ma": null,
58
+ "n_success": 66,
59
+ "policy_version": 18,
60
+ "rl_env_steps": 143879,
61
+ "round": 18,
62
+ "seed_start": 960000,
63
+ "sr": 0.33,
64
+ "success_requested": 0,
65
+ "success_requested_imitation": 0,
66
+ "success_rounds_with_skips": 0,
67
+ "success_skipped": 0,
68
+ "success_skipped_imitation": 0,
69
+ "success_skipped_td": 0,
70
+ "target_step": 140000,
71
+ "update_index": 54000
72
+ }
73
+ }
74
+ },
75
+ "base_bc": {
76
+ "repo_id": "yangfy0627/RL-postTrain",
77
+ "revision": "95320f4b68a221fc235e2897a21558fa43545015",
78
+ "path": "Robomimic/study1/ToolHang/mixed100/seed_1/checkpoints/epoch_300.pkl",
79
+ "sha256": "0fb257b0b57fbf50b14ac849364a2c29db4ffb32468ab4c36a716fd0468ea7b8"
80
+ },
81
+ "export_scope": "learner state checkpoints, original configuration, provenance and evaluation records; replay segments and offline_buffer.pkl remain in source runs",
82
+ "best_selection": "maximum observed periodic success rate on 200 episodes, earliest tie",
83
+ "prepared_at": "2026-10-26T20:55:55.938473+00:00"
84
+ }
85
+ ]
86
+ }