Archive complete RL run 23871f6e with replay, checkpoints, diagnostics and provenance
Browse files- RL/ToolHang100/QC/diverse100/seed_1/best/source_metadata.json +27 -0
- RL/ToolHang100/QC/diverse100/seed_1/best/source_resolved_config.json +1195 -0
- RL/ToolHang100/QC/diverse100/seed_1/best/state.pkl +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/comparison_evaluations.jsonl +81 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/README.md +5 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/SHA256SUMS +27 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/archive.json +33 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part000 +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part001 +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part002 +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part003 +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part004 +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/local_archive_verification.json +6 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/manifest.json +0 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/publication.json +62 -0
- RL/ToolHang100/QC/diverse100/seed_1/complete/restore.py +38 -0
- RL/ToolHang100/QC/diverse100/seed_1/convergence.json +367 -0
- RL/ToolHang100/QC/diverse100/seed_1/convergence.jsonl +81 -0
- RL/ToolHang100/QC/diverse100/seed_1/eval/final.parquet +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/final/source_metadata.json +314 -0
- RL/ToolHang100/QC/diverse100/seed_1/final/source_resolved_config.json +1195 -0
- RL/ToolHang100/QC/diverse100/seed_1/final/state.pkl +3 -0
- RL/ToolHang100/QC/diverse100/seed_1/frozen_base.json +6 -0
- RL/ToolHang100/QC/diverse100/seed_1/offline_buffer.json +847 -0
- RL/ToolHang100/QC/diverse100/seed_1/resolved_config.json +1195 -0
- RL/ToolHang100/QC/diverse100/seed_1/start_protocol.json +153 -0
- RL/ToolHang100/QC/diverse100/seed_1/status.json +174 -0
- RL/ToolHang100/QC/diverse100/seed_1/training_accounting.json +51 -0
RL/ToolHang100/QC/diverse100/seed_1/best/source_metadata.json
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint_sha256": "fc32c5a044147dca30b67057fc8d732e1a35ad69786d41a690ce1977cff0bb57",
|
| 3 |
+
"evaluation": {
|
| 4 |
+
"chunk_transitions": 11000,
|
| 5 |
+
"episodes": 200,
|
| 6 |
+
"eval_index": 4,
|
| 7 |
+
"gain": null,
|
| 8 |
+
"gradient_updates": 33000,
|
| 9 |
+
"ma": null,
|
| 10 |
+
"n_success": 67,
|
| 11 |
+
"policy_version": 11,
|
| 12 |
+
"rl_env_steps": 87901,
|
| 13 |
+
"round": 11,
|
| 14 |
+
"seed_start": 960000,
|
| 15 |
+
"sr": 0.335,
|
| 16 |
+
"success_requested": 0,
|
| 17 |
+
"success_requested_imitation": 0,
|
| 18 |
+
"success_rounds_with_skips": 0,
|
| 19 |
+
"success_skipped": 0,
|
| 20 |
+
"success_skipped_imitation": 0,
|
| 21 |
+
"success_skipped_td": 0,
|
| 22 |
+
"target_step": 80000,
|
| 23 |
+
"update_index": 33000
|
| 24 |
+
},
|
| 25 |
+
"includes": "complete learner state, optimizer, targets, RNG and replay",
|
| 26 |
+
"selection": "observed_periodic_sr"
|
| 27 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/best/source_resolved_config.json
ADDED
|
@@ -0,0 +1,1195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
|
| 3 |
+
"env": {
|
| 4 |
+
"chunk_length": 8,
|
| 5 |
+
"n_envs": 4
|
| 6 |
+
},
|
| 7 |
+
"learner": {
|
| 8 |
+
"convention": "unified_env_step",
|
| 9 |
+
"gamma_env": 0.999
|
| 10 |
+
},
|
| 11 |
+
"overrides": [
|
| 12 |
+
{
|
| 13 |
+
"decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
|
| 14 |
+
"key": "n_step_chunks",
|
| 15 |
+
"reason": "task-specific unified Study2 setting",
|
| 16 |
+
"upstream": 3,
|
| 17 |
+
"value": 5
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 21 |
+
"key": "offline.cohort_size",
|
| 22 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 23 |
+
"upstream": 50,
|
| 24 |
+
"value": 100
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 28 |
+
"key": "offline.episodes",
|
| 29 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 30 |
+
"upstream": 50,
|
| 31 |
+
"value": 100
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"decision_ref": "[0930-S2DEC]",
|
| 35 |
+
"key": "learner.qc.agent.act_bounds_json",
|
| 36 |
+
"reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
|
| 37 |
+
"upstream": null,
|
| 38 |
+
"value": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"decision_ref": "[0930-S2DEC]",
|
| 42 |
+
"key": "learner.qc.agent.act_max",
|
| 43 |
+
"reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
|
| 44 |
+
"upstream": 1.0,
|
| 45 |
+
"value": 3.0058202743530273
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"decision_ref": "[0930-S2DEC]",
|
| 49 |
+
"key": "learner.qc.agent.act_min",
|
| 50 |
+
"reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
|
| 51 |
+
"upstream": -1.0,
|
| 52 |
+
"value": -2.9472272396087646
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"decision_ref": "[0930-S2DEC]",
|
| 56 |
+
"key": "learner.qc.agent.actor_lr_partition",
|
| 57 |
+
"reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
|
| 58 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 59 |
+
"value": 0.0001
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 63 |
+
"key": "learner.qc.agent.actor_num_samples",
|
| 64 |
+
"reason": "single BoN knob including QC TD",
|
| 65 |
+
"upstream": 32,
|
| 66 |
+
"value": 16
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 70 |
+
"key": "learner.qc.agent.best_of_n",
|
| 71 |
+
"reason": "single BoN knob: collection/eval/QC TD",
|
| 72 |
+
"upstream": 8,
|
| 73 |
+
"value": 16
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 77 |
+
"key": "learner.qc.agent.bon_agg",
|
| 78 |
+
"reason": "Best-of-N selection by the ensemble mean",
|
| 79 |
+
"upstream": null,
|
| 80 |
+
"value": "mean"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 84 |
+
"key": "learner.qc.agent.bon_sft",
|
| 85 |
+
"reason": "variant qc flag bon_sft",
|
| 86 |
+
"upstream": "upstream_yaml",
|
| 87 |
+
"value": false
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"decision_ref": "[0930-S2DEC]",
|
| 91 |
+
"key": "learner.qc.agent.critic_lr_partition",
|
| 92 |
+
"reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
|
| 93 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 94 |
+
"value": 0.0003
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 98 |
+
"key": "learner.qc.agent.partition_optimizer",
|
| 99 |
+
"reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
|
| 100 |
+
"upstream": null,
|
| 101 |
+
"value": true
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 105 |
+
"key": "learner.qc.agent.q_target_agg",
|
| 106 |
+
"reason": "REDQ-style TD target: min over a random critic subset",
|
| 107 |
+
"upstream": null,
|
| 108 |
+
"value": "subset_min"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 112 |
+
"key": "learner.qc.agent.q_target_replace",
|
| 113 |
+
"reason": "subset drawn without replacement",
|
| 114 |
+
"upstream": null,
|
| 115 |
+
"value": false
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 119 |
+
"key": "learner.qc.agent.q_target_subset",
|
| 120 |
+
"reason": "TD-target subset size",
|
| 121 |
+
"upstream": null,
|
| 122 |
+
"value": 2
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"decision_ref": "[0930-S2DEC]",
|
| 126 |
+
"key": "learner.qc.agent.tau",
|
| 127 |
+
"reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
|
| 128 |
+
"upstream": 0.05,
|
| 129 |
+
"value": 0.005
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"decision_ref": "[0930-S2DEC]",
|
| 133 |
+
"key": "learner.qc.agent.value_hidden_dims",
|
| 134 |
+
"reason": "unified.critic_hidden (critics only; actors native)",
|
| 135 |
+
"upstream": [
|
| 136 |
+
512,
|
| 137 |
+
512,
|
| 138 |
+
512,
|
| 139 |
+
512,
|
| 140 |
+
512
|
| 141 |
+
],
|
| 142 |
+
"value": [
|
| 143 |
+
512,
|
| 144 |
+
512,
|
| 145 |
+
512
|
| 146 |
+
]
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 150 |
+
"key": "learner.qc.training.bc_pi_steps",
|
| 151 |
+
"reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
|
| 152 |
+
"upstream": 1000000,
|
| 153 |
+
"value": 0
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 157 |
+
"key": "learner.qc.loop.buffer.utd_online",
|
| 158 |
+
"reason": "UTD=3 per executed action chunk",
|
| 159 |
+
"upstream": 1,
|
| 160 |
+
"value": "3 gradient updates per executed chunk"
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 164 |
+
"key": "learner.qc.loop.n_step",
|
| 165 |
+
"reason": "n-step=3 counted in action chunks",
|
| 166 |
+
"upstream": "1 chunk (sequence_length = horizon_length)",
|
| 167 |
+
"value": "5 executed chunks (unified_env_step window)"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 171 |
+
"key": "learner.qc.loop.critic_actor_ratio",
|
| 172 |
+
"reason": "critic every update, actor every 2nd global update",
|
| 173 |
+
"upstream": "1:1 (utd_q = utd_pi)",
|
| 174 |
+
"value": "2:1"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 178 |
+
"key": "learner.qc.loop.buffer.start_training",
|
| 179 |
+
"reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
|
| 180 |
+
"upstream": 10000,
|
| 181 |
+
"value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 185 |
+
"key": "learner.qc.loop.buffer.offline_ratio",
|
| 186 |
+
"reason": "[1017-S2OFFBC] adaptive_ratio off",
|
| 187 |
+
"upstream": 0.0,
|
| 188 |
+
"value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 192 |
+
"key": "learner.qc.loop.buffer.size",
|
| 193 |
+
"reason": "one shared replay of executed chunks; success buffer = mask",
|
| 194 |
+
"upstream": 2000000,
|
| 195 |
+
"value": "1000000 executed chunks, shared FIFO"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 199 |
+
"key": "learner.qc.actor_architecture",
|
| 200 |
+
"reason": "Study2 fine-tunes the Study1 pixel_state CFM",
|
| 201 |
+
"upstream": [
|
| 202 |
+
512,
|
| 203 |
+
512,
|
| 204 |
+
512,
|
| 205 |
+
512,
|
| 206 |
+
512
|
| 207 |
+
],
|
| 208 |
+
"value": "checkpoint StandardGenerator (EMA generator copy)"
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 212 |
+
"key": "learner.qc.observation",
|
| 213 |
+
"reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
|
| 214 |
+
"upstream": "low-dim state",
|
| 215 |
+
"value": "frozen Ri features c [137] from the checkpoint encoder"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"decision_ref": "[0930-S2DEC]",
|
| 219 |
+
"key": "learner.qc.reward",
|
| 220 |
+
"reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
|
| 221 |
+
"upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
|
| 222 |
+
"value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"decision_ref": "[0930-S2DEC]",
|
| 226 |
+
"key": "learner.qc.td_target",
|
| 227 |
+
"reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
|
| 228 |
+
"upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
|
| 229 |
+
"value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"decision_ref": "[0930-S2DEC]",
|
| 233 |
+
"key": "learner.qc.eval",
|
| 234 |
+
"reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
|
| 235 |
+
"upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
|
| 236 |
+
"value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 240 |
+
"key": "learner.qc.lr_schedule",
|
| 241 |
+
"reason": "RL env-step clock shared with extra success TD",
|
| 242 |
+
"upstream": "native per-optimizer schedule",
|
| 243 |
+
"value": {
|
| 244 |
+
"actor": 0.0001,
|
| 245 |
+
"critic": 0.0003,
|
| 246 |
+
"decay_end_env_steps": 1606196,
|
| 247 |
+
"min_ratio": 0.1,
|
| 248 |
+
"schedule": "warmup_cosine",
|
| 249 |
+
"temperature": 0.0003,
|
| 250 |
+
"warmup_env_steps": 20000
|
| 251 |
+
}
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"decision_ref": "[0930-S2DEC]",
|
| 255 |
+
"key": "learner.qc.agent.q_min_q_max",
|
| 256 |
+
"reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
|
| 257 |
+
"upstream": [
|
| 258 |
+
-999.9999999999991,
|
| 259 |
+
0.0
|
| 260 |
+
],
|
| 261 |
+
"value": [
|
| 262 |
+
0.0,
|
| 263 |
+
1.0
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
],
|
| 267 |
+
"protocol": {
|
| 268 |
+
"checkpoint_every_rounds": 5,
|
| 269 |
+
"checkpoint_retention": {
|
| 270 |
+
"best": true,
|
| 271 |
+
"collapse_drop": {
|
| 272 |
+
"min_abs": 0.2,
|
| 273 |
+
"ratio": 0.5
|
| 274 |
+
},
|
| 275 |
+
"diagnostics": true,
|
| 276 |
+
"every_evals": 3
|
| 277 |
+
},
|
| 278 |
+
"collection": {
|
| 279 |
+
"n_envs": 4,
|
| 280 |
+
"train_seed_base": 100000000,
|
| 281 |
+
"train_seed_stride": 1000000,
|
| 282 |
+
"training_freq_chunks": 1000
|
| 283 |
+
},
|
| 284 |
+
"convergence": {
|
| 285 |
+
"early_stop": false,
|
| 286 |
+
"gain_lag": 10,
|
| 287 |
+
"high_sr": 0.95,
|
| 288 |
+
"high_sr_evals": 5,
|
| 289 |
+
"ma_window": 10,
|
| 290 |
+
"max_env_steps": 1606196,
|
| 291 |
+
"min_env_steps": 200000,
|
| 292 |
+
"min_gain": 0.02,
|
| 293 |
+
"zero_success_evals": 8
|
| 294 |
+
},
|
| 295 |
+
"critic_ensemble": {
|
| 296 |
+
"num": 10,
|
| 297 |
+
"policy": "mean",
|
| 298 |
+
"target": {
|
| 299 |
+
"kind": "subset_min",
|
| 300 |
+
"replace": false,
|
| 301 |
+
"size": 2
|
| 302 |
+
}
|
| 303 |
+
},
|
| 304 |
+
"eval": {
|
| 305 |
+
"best_of_n": 16,
|
| 306 |
+
"episodes": 200,
|
| 307 |
+
"every_env_steps": 20000,
|
| 308 |
+
"include_step0": true,
|
| 309 |
+
"seed_start": 960000,
|
| 310 |
+
"workers": 4
|
| 311 |
+
},
|
| 312 |
+
"final_eval": {
|
| 313 |
+
"episodes": 300,
|
| 314 |
+
"on_zero_stop": true,
|
| 315 |
+
"seed_start": 800000
|
| 316 |
+
},
|
| 317 |
+
"n_step_chunks": 5,
|
| 318 |
+
"offline": {
|
| 319 |
+
"adaptive_ratio": false,
|
| 320 |
+
"cohort_size": 100,
|
| 321 |
+
"episodes": 100,
|
| 322 |
+
"learners_without_offline": [
|
| 323 |
+
"dsrl_sac"
|
| 324 |
+
],
|
| 325 |
+
"ratio": {
|
| 326 |
+
"decay_env_steps": {
|
| 327 |
+
"can": 160000,
|
| 328 |
+
"square": 320000,
|
| 329 |
+
"tool_hang": 640000,
|
| 330 |
+
"transport": 320000
|
| 331 |
+
},
|
| 332 |
+
"end": 0.1,
|
| 333 |
+
"start": {
|
| 334 |
+
"can": 0.7,
|
| 335 |
+
"square": 0.5,
|
| 336 |
+
"tool_hang": 0.5,
|
| 337 |
+
"transport": 0.9
|
| 338 |
+
}
|
| 339 |
+
},
|
| 340 |
+
"selection": "ascending_demo_index",
|
| 341 |
+
"source": "bc_regime",
|
| 342 |
+
"unsupported_learners": [
|
| 343 |
+
"zprl"
|
| 344 |
+
]
|
| 345 |
+
},
|
| 346 |
+
"replay": {
|
| 347 |
+
"capacity_chunks": 1000000,
|
| 348 |
+
"pin_start_successes": false,
|
| 349 |
+
"success_buffer": "success_mask_on_shared_fifo",
|
| 350 |
+
"success_skip_rule": "skip_below_success_batch_size",
|
| 351 |
+
"unit": "executed_chunk"
|
| 352 |
+
},
|
| 353 |
+
"start": {
|
| 354 |
+
"max_episodes": 200,
|
| 355 |
+
"mode": "offline_buffer",
|
| 356 |
+
"policy": {
|
| 357 |
+
"default": "base_cfm_ode",
|
| 358 |
+
"zprl": "vib_base"
|
| 359 |
+
},
|
| 360 |
+
"target_successes": 20
|
| 361 |
+
},
|
| 362 |
+
"unified": {
|
| 363 |
+
"actor_target_tau": 0.005,
|
| 364 |
+
"critic_hidden": [
|
| 365 |
+
512,
|
| 366 |
+
512,
|
| 367 |
+
512
|
| 368 |
+
],
|
| 369 |
+
"critic_target_every_critic_step": true,
|
| 370 |
+
"critic_target_tau": 0.005,
|
| 371 |
+
"dsrl_noise_imitation_reference_dims": 28,
|
| 372 |
+
"gamma_env": 0.999,
|
| 373 |
+
"lr": {
|
| 374 |
+
"actor": 0.0001,
|
| 375 |
+
"critic": 0.0003,
|
| 376 |
+
"decay_end_env_steps": 1606196,
|
| 377 |
+
"min_ratio": 0.1,
|
| 378 |
+
"schedule": "warmup_cosine",
|
| 379 |
+
"temperature": 0.0003,
|
| 380 |
+
"warmup_env_steps": 20000
|
| 381 |
+
},
|
| 382 |
+
"reward": "binary_first_success",
|
| 383 |
+
"success_td_polyak": true,
|
| 384 |
+
"timeout_bootstrap": false,
|
| 385 |
+
"zprl_imitation": "mse_pretanh_mean"
|
| 386 |
+
},
|
| 387 |
+
"updates": {
|
| 388 |
+
"batch_size": 256,
|
| 389 |
+
"critic_per_actor": 2,
|
| 390 |
+
"success_batch_size": 256,
|
| 391 |
+
"utd_per_chunk": 3
|
| 392 |
+
},
|
| 393 |
+
"version": "study2_protocol_v10"
|
| 394 |
+
},
|
| 395 |
+
"protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
|
| 396 |
+
"provenance_extra": {
|
| 397 |
+
"agent_library": {
|
| 398 |
+
"framework": "jax",
|
| 399 |
+
"sources": {
|
| 400 |
+
"__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
|
| 401 |
+
"_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 402 |
+
"_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 403 |
+
"_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 404 |
+
"_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 405 |
+
"_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
|
| 406 |
+
"_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
|
| 407 |
+
"_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
|
| 408 |
+
"_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
|
| 409 |
+
"_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
|
| 410 |
+
"_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
|
| 411 |
+
"_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
|
| 412 |
+
"_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
|
| 413 |
+
"_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
|
| 414 |
+
"_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
|
| 415 |
+
"_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
|
| 416 |
+
"_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
|
| 417 |
+
"_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
|
| 418 |
+
"_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 419 |
+
"_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
|
| 420 |
+
"_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
|
| 421 |
+
"_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
|
| 422 |
+
"_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
|
| 423 |
+
"_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
|
| 424 |
+
"_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
|
| 425 |
+
"_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
|
| 426 |
+
"_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
|
| 427 |
+
"_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
|
| 428 |
+
"_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
|
| 429 |
+
"_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
|
| 430 |
+
"_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
|
| 431 |
+
"dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
|
| 432 |
+
"dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
|
| 433 |
+
"dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
|
| 434 |
+
"factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
|
| 435 |
+
"flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
|
| 436 |
+
"native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
|
| 437 |
+
"native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
|
| 438 |
+
"native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
|
| 439 |
+
"native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
|
| 440 |
+
"native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
|
| 441 |
+
"networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
|
| 442 |
+
"ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
|
| 443 |
+
"optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
|
| 444 |
+
"qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
|
| 445 |
+
"registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
|
| 446 |
+
"residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
|
| 447 |
+
"telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
|
| 448 |
+
"train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
|
| 449 |
+
"vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
|
| 450 |
+
"vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
|
| 451 |
+
"zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
|
| 452 |
+
},
|
| 453 |
+
"version": "study2_jax_agents_v2"
|
| 454 |
+
},
|
| 455 |
+
"base_framework": "jax",
|
| 456 |
+
"bc_identity": {
|
| 457 |
+
"architecture": "dice_vision_resnet_flow_unet_v1",
|
| 458 |
+
"family": "StandardBaseBC",
|
| 459 |
+
"framework": "jax"
|
| 460 |
+
},
|
| 461 |
+
"checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261024_toolhang_diverse100_v10_three/base/diverse100/epoch_300.pkl",
|
| 462 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 463 |
+
"experiment": "tool_hang",
|
| 464 |
+
"frozen_hash": "719a7185a0d6a4877c76a5e1d9b1adb163a3d863d28bced54e79d534e8db6873",
|
| 465 |
+
"learner": {
|
| 466 |
+
"action_bounds": {
|
| 467 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 468 |
+
"compat_act_max": 3.0058202743530273,
|
| 469 |
+
"compat_act_min": -2.9472272396087646,
|
| 470 |
+
"hi": [
|
| 471 |
+
1.0,
|
| 472 |
+
1.0,
|
| 473 |
+
1.0,
|
| 474 |
+
3.0058202743530273,
|
| 475 |
+
1.8893057107925415,
|
| 476 |
+
1.0,
|
| 477 |
+
1.0
|
| 478 |
+
],
|
| 479 |
+
"kind": "per_dimension",
|
| 480 |
+
"lo": [
|
| 481 |
+
-1.0,
|
| 482 |
+
-1.0,
|
| 483 |
+
-1.0,
|
| 484 |
+
-2.9472272396087646,
|
| 485 |
+
-1.9721344709396362,
|
| 486 |
+
-1.318867564201355,
|
| 487 |
+
-1.0
|
| 488 |
+
],
|
| 489 |
+
"note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
|
| 490 |
+
},
|
| 491 |
+
"agent_config": {
|
| 492 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 493 |
+
"act_max": 3.0058202743530273,
|
| 494 |
+
"act_min": -2.9472272396087646,
|
| 495 |
+
"action_chunking": true,
|
| 496 |
+
"action_critic_hidden_dims": null,
|
| 497 |
+
"action_dim": 7,
|
| 498 |
+
"action_num_qs": 10,
|
| 499 |
+
"action_repeat": false,
|
| 500 |
+
"actor_backbone": "mlp",
|
| 501 |
+
"actor_decay_steps": 50000,
|
| 502 |
+
"actor_end_value": 2e-05,
|
| 503 |
+
"actor_hidden_dims": [
|
| 504 |
+
512,
|
| 505 |
+
512,
|
| 506 |
+
512,
|
| 507 |
+
512,
|
| 508 |
+
512
|
| 509 |
+
],
|
| 510 |
+
"actor_layer_norm": false,
|
| 511 |
+
"actor_lr": 0.0003,
|
| 512 |
+
"actor_lr_partition": 0.0001,
|
| 513 |
+
"actor_num_samples": 16,
|
| 514 |
+
"actor_obs": "state",
|
| 515 |
+
"actor_scheduler": "cosine",
|
| 516 |
+
"actor_tau": 0.005,
|
| 517 |
+
"actor_type": "best-of-n",
|
| 518 |
+
"actor_warmup_steps": 2000,
|
| 519 |
+
"actor_weight_decay": 0.0,
|
| 520 |
+
"adv_clip_min": null,
|
| 521 |
+
"agent_name": "acfql",
|
| 522 |
+
"alpha": 500,
|
| 523 |
+
"batch_size": 256,
|
| 524 |
+
"best_of_n": 16,
|
| 525 |
+
"bon_agg": "mean",
|
| 526 |
+
"bon_sft": false,
|
| 527 |
+
"bon_sft_coeff": 1.0,
|
| 528 |
+
"bootstrap_ratio": 0.5,
|
| 529 |
+
"bptt_num_samples": 4,
|
| 530 |
+
"calql_q_subsample": 2,
|
| 531 |
+
"clip_grad_norm": 1000.0,
|
| 532 |
+
"clip_intermediate_actions": true,
|
| 533 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 534 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 535 |
+
"constant_noise_std": 0.01,
|
| 536 |
+
"cql_alpha": 5.0,
|
| 537 |
+
"cql_n_actions": 10,
|
| 538 |
+
"cql_temp": 1.0,
|
| 539 |
+
"critic_backbone": "mlp",
|
| 540 |
+
"critic_decay_steps": 5000,
|
| 541 |
+
"critic_end_value": 1e-08,
|
| 542 |
+
"critic_loss_type": "mse",
|
| 543 |
+
"critic_lr": 0.0003,
|
| 544 |
+
"critic_lr_partition": 0.0003,
|
| 545 |
+
"critic_obs": "state",
|
| 546 |
+
"critic_scheduler": "constant",
|
| 547 |
+
"critic_warmup_steps": 500,
|
| 548 |
+
"critic_weight_decay": 1e-05,
|
| 549 |
+
"denoised_clip_value": 1.0,
|
| 550 |
+
"discount": 0.999,
|
| 551 |
+
"edit_action_scale": 0.5,
|
| 552 |
+
"edit_actor_hidden_dims": null,
|
| 553 |
+
"edit_hidden_dims": [
|
| 554 |
+
256,
|
| 555 |
+
256,
|
| 556 |
+
256
|
| 557 |
+
],
|
| 558 |
+
"edit_init_temperature": 1.0,
|
| 559 |
+
"edit_layer_norm": true,
|
| 560 |
+
"edit_lr": 0.0003,
|
| 561 |
+
"edit_policy_bound": 0.1,
|
| 562 |
+
"edit_start_step": 100000,
|
| 563 |
+
"edit_target_entropy": null,
|
| 564 |
+
"edit_weight_decay": 0.0,
|
| 565 |
+
"encoder": null,
|
| 566 |
+
"entropy_backup": false,
|
| 567 |
+
"flow_steps": 10,
|
| 568 |
+
"fql_distillation_coeff": 10.0,
|
| 569 |
+
"fql_train_main_policy": true,
|
| 570 |
+
"freeze_vision_encoder": false,
|
| 571 |
+
"ft_flow_steps": 10,
|
| 572 |
+
"horizon_length": 8,
|
| 573 |
+
"init_temperature": 1.0,
|
| 574 |
+
"layer_norm": true,
|
| 575 |
+
"lr": 0.0003,
|
| 576 |
+
"lr_action_critic": null,
|
| 577 |
+
"lr_actor": null,
|
| 578 |
+
"lr_actor_bc_flow": null,
|
| 579 |
+
"lr_critic": null,
|
| 580 |
+
"lr_edit_actor": null,
|
| 581 |
+
"lr_noise_critic": null,
|
| 582 |
+
"max_noise_std": 0.01,
|
| 583 |
+
"mc_regression": false,
|
| 584 |
+
"mc_regression_coeff": 1.0,
|
| 585 |
+
"min_noise_std": 0.01,
|
| 586 |
+
"mip_q_ensemble": false,
|
| 587 |
+
"mip_q_noise_scale": 1.0,
|
| 588 |
+
"mip_q_t_star": 0.9,
|
| 589 |
+
"mip_t_star": 0.999,
|
| 590 |
+
"muon_beta": 0.95,
|
| 591 |
+
"muon_nesterov": true,
|
| 592 |
+
"muon_ns_steps": 5,
|
| 593 |
+
"n_action_samples": 8,
|
| 594 |
+
"n_edit_samples": 8,
|
| 595 |
+
"noise_critic_grad_steps": 1,
|
| 596 |
+
"noise_dim": null,
|
| 597 |
+
"noise_net_hidden_dims": [
|
| 598 |
+
256,
|
| 599 |
+
256
|
| 600 |
+
],
|
| 601 |
+
"noise_net_layer_norm": true,
|
| 602 |
+
"normalize_act_space_dimension": false,
|
| 603 |
+
"normalize_denoising_horizon": false,
|
| 604 |
+
"normalize_q_loss": false,
|
| 605 |
+
"num_bins": 256,
|
| 606 |
+
"num_ensemble_members": 10,
|
| 607 |
+
"num_qs": 10,
|
| 608 |
+
"ob_dims": [
|
| 609 |
+
137
|
| 610 |
+
],
|
| 611 |
+
"one_step_decay_steps": null,
|
| 612 |
+
"one_step_end_value": null,
|
| 613 |
+
"one_step_hidden_dims": [
|
| 614 |
+
512,
|
| 615 |
+
512,
|
| 616 |
+
512,
|
| 617 |
+
512
|
| 618 |
+
],
|
| 619 |
+
"one_step_layer_norm": false,
|
| 620 |
+
"one_step_lr": null,
|
| 621 |
+
"one_step_scheduler": null,
|
| 622 |
+
"one_step_warmup_steps": null,
|
| 623 |
+
"optimizer_action_critic": null,
|
| 624 |
+
"optimizer_actor": "adam",
|
| 625 |
+
"optimizer_actor_bc_flow": "adam",
|
| 626 |
+
"optimizer_critic": "adam",
|
| 627 |
+
"optimizer_edit_actor": null,
|
| 628 |
+
"paligemma_tokenizer_path": null,
|
| 629 |
+
"partition_optimizer": true,
|
| 630 |
+
"policy_type": "flow",
|
| 631 |
+
"ppo_batch_size": 256,
|
| 632 |
+
"ppo_lr": 4.5e-05,
|
| 633 |
+
"proprio_dim": null,
|
| 634 |
+
"q_agg": "mean",
|
| 635 |
+
"q_max": 1.0,
|
| 636 |
+
"q_min": 0.0,
|
| 637 |
+
"q_target_agg": "subset_min",
|
| 638 |
+
"q_target_replace": false,
|
| 639 |
+
"q_target_subset": 2,
|
| 640 |
+
"q_variance_reduction": false,
|
| 641 |
+
"q_vr_num_samples": 8,
|
| 642 |
+
"q_vr_reduction": "mean",
|
| 643 |
+
"randn_clip_value": 3.0,
|
| 644 |
+
"shortcut_denoising_steps": 4,
|
| 645 |
+
"shortcut_inference_steps": 4,
|
| 646 |
+
"simba_actor_hidden_dims": [
|
| 647 |
+
1024,
|
| 648 |
+
1024
|
| 649 |
+
],
|
| 650 |
+
"simba_critic_hidden_dims": [
|
| 651 |
+
1024,
|
| 652 |
+
1024,
|
| 653 |
+
1024
|
| 654 |
+
],
|
| 655 |
+
"subsample_bon": false,
|
| 656 |
+
"target_entropy": null,
|
| 657 |
+
"tau": 0.005,
|
| 658 |
+
"tf_pi_conditioning": "adaln",
|
| 659 |
+
"tf_pi_dropout": 0.0,
|
| 660 |
+
"tf_pi_embed_dim": 256,
|
| 661 |
+
"tf_pi_heads": 4,
|
| 662 |
+
"tf_pi_layers": 8,
|
| 663 |
+
"tf_q_conditioning": "adaln",
|
| 664 |
+
"tf_q_dropout": 0.0,
|
| 665 |
+
"tf_q_embed_dim": 256,
|
| 666 |
+
"tf_q_heads": 4,
|
| 667 |
+
"tf_q_layers": 8,
|
| 668 |
+
"time_embedding": "sinusoidal",
|
| 669 |
+
"time_embedding_dim": 32,
|
| 670 |
+
"train_one_step_in_offline": true,
|
| 671 |
+
"train_one_step_in_online": true,
|
| 672 |
+
"useFiLM": false,
|
| 673 |
+
"useSimBa": false,
|
| 674 |
+
"useTF": false,
|
| 675 |
+
"use_constant_noise": true,
|
| 676 |
+
"use_constant_scheduler_for_bc": true,
|
| 677 |
+
"use_denoiser": false,
|
| 678 |
+
"use_edit_actor": false,
|
| 679 |
+
"use_mip_q": false,
|
| 680 |
+
"use_muon": false,
|
| 681 |
+
"use_one_step_policy": false,
|
| 682 |
+
"use_shortcut": false,
|
| 683 |
+
"use_state": "proprio",
|
| 684 |
+
"use_success_buffer_q": false,
|
| 685 |
+
"value_hidden_dims": [
|
| 686 |
+
512,
|
| 687 |
+
512,
|
| 688 |
+
512
|
| 689 |
+
],
|
| 690 |
+
"vit_encoder": "minvit",
|
| 691 |
+
"vit_img_proj_dim": 0,
|
| 692 |
+
"vit_state_proj_dim": 0
|
| 693 |
+
},
|
| 694 |
+
"convention": "unified_env_step",
|
| 695 |
+
"eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
|
| 696 |
+
"gamma_env": 0.999,
|
| 697 |
+
"gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
|
| 698 |
+
"learner": "qc",
|
| 699 |
+
"optimizers": {
|
| 700 |
+
"actor_lr": 0.0001,
|
| 701 |
+
"critic_lr": 0.0003,
|
| 702 |
+
"reset_optimizers_with_lr": "not used",
|
| 703 |
+
"schedule": {
|
| 704 |
+
"actor": 0.0001,
|
| 705 |
+
"critic": 0.0003,
|
| 706 |
+
"decay_end_env_steps": 1606196,
|
| 707 |
+
"min_ratio": 0.1,
|
| 708 |
+
"schedule": "warmup_cosine",
|
| 709 |
+
"temperature": 0.0003,
|
| 710 |
+
"warmup_env_steps": 20000
|
| 711 |
+
}
|
| 712 |
+
},
|
| 713 |
+
"qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
|
| 714 |
+
"recipe": {
|
| 715 |
+
"derived": {
|
| 716 |
+
"agent.actor_num_samples": {
|
| 717 |
+
"after": 8,
|
| 718 |
+
"before": 32
|
| 719 |
+
},
|
| 720 |
+
"agent.actor_type": {
|
| 721 |
+
"after": "best-of-n",
|
| 722 |
+
"before": "upstream_yaml"
|
| 723 |
+
},
|
| 724 |
+
"agent.bon_sft": {
|
| 725 |
+
"after": false,
|
| 726 |
+
"before": "upstream_yaml"
|
| 727 |
+
}
|
| 728 |
+
},
|
| 729 |
+
"explicit_overrides": {},
|
| 730 |
+
"source": "third_party/ogpo/scripts/qc/toolhang.sh",
|
| 731 |
+
"upstream_exact": false
|
| 732 |
+
},
|
| 733 |
+
"reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
|
| 734 |
+
"rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
|
| 735 |
+
"scan_block": 100,
|
| 736 |
+
"start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
|
| 737 |
+
"success_terms": {
|
| 738 |
+
"bc_on_success_batch": false,
|
| 739 |
+
"critic_success_td": false,
|
| 740 |
+
"missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
|
| 741 |
+
},
|
| 742 |
+
"td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
|
| 743 |
+
"unified_notes": {
|
| 744 |
+
"critic_target_every_critic_step": {
|
| 745 |
+
"note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
|
| 746 |
+
"value": true
|
| 747 |
+
},
|
| 748 |
+
"success_td_polyak": {
|
| 749 |
+
"note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
|
| 750 |
+
"value": true
|
| 751 |
+
}
|
| 752 |
+
},
|
| 753 |
+
"upstream_config": {
|
| 754 |
+
"agent": {
|
| 755 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 756 |
+
"act_max": 3.0058202743530273,
|
| 757 |
+
"act_min": -2.9472272396087646,
|
| 758 |
+
"action_chunking": true,
|
| 759 |
+
"action_critic_hidden_dims": null,
|
| 760 |
+
"action_dim": null,
|
| 761 |
+
"action_num_qs": 10,
|
| 762 |
+
"action_repeat": false,
|
| 763 |
+
"actor_backbone": "mlp",
|
| 764 |
+
"actor_decay_steps": 50000,
|
| 765 |
+
"actor_end_value": 2e-05,
|
| 766 |
+
"actor_hidden_dims": [
|
| 767 |
+
512,
|
| 768 |
+
512,
|
| 769 |
+
512,
|
| 770 |
+
512,
|
| 771 |
+
512
|
| 772 |
+
],
|
| 773 |
+
"actor_layer_norm": false,
|
| 774 |
+
"actor_lr": 0.0003,
|
| 775 |
+
"actor_lr_partition": 0.0001,
|
| 776 |
+
"actor_num_samples": 16,
|
| 777 |
+
"actor_obs": "state",
|
| 778 |
+
"actor_scheduler": "cosine",
|
| 779 |
+
"actor_tau": 0.005,
|
| 780 |
+
"actor_type": "best-of-n",
|
| 781 |
+
"actor_warmup_steps": 2000,
|
| 782 |
+
"actor_weight_decay": 0.0,
|
| 783 |
+
"adv_clip_min": null,
|
| 784 |
+
"agent_name": "acfql",
|
| 785 |
+
"alpha": 500,
|
| 786 |
+
"batch_size": 256,
|
| 787 |
+
"best_of_n": 16,
|
| 788 |
+
"bon_agg": "mean",
|
| 789 |
+
"bon_sft": false,
|
| 790 |
+
"bon_sft_coeff": 1.0,
|
| 791 |
+
"bootstrap_ratio": 0.5,
|
| 792 |
+
"bptt_num_samples": 4,
|
| 793 |
+
"calql_q_subsample": 2,
|
| 794 |
+
"clip_grad_norm": 1000.0,
|
| 795 |
+
"clip_intermediate_actions": true,
|
| 796 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 797 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 798 |
+
"constant_noise_std": 0.01,
|
| 799 |
+
"cql_alpha": 5.0,
|
| 800 |
+
"cql_n_actions": 10,
|
| 801 |
+
"cql_temp": 1.0,
|
| 802 |
+
"critic_backbone": "mlp",
|
| 803 |
+
"critic_decay_steps": 5000,
|
| 804 |
+
"critic_end_value": 1e-08,
|
| 805 |
+
"critic_loss_type": "mse",
|
| 806 |
+
"critic_lr": 0.0003,
|
| 807 |
+
"critic_lr_partition": 0.0003,
|
| 808 |
+
"critic_obs": "state",
|
| 809 |
+
"critic_scheduler": "constant",
|
| 810 |
+
"critic_warmup_steps": 500,
|
| 811 |
+
"critic_weight_decay": 1e-05,
|
| 812 |
+
"denoised_clip_value": 1.0,
|
| 813 |
+
"discount": 0.999,
|
| 814 |
+
"edit_action_scale": 0.5,
|
| 815 |
+
"edit_actor_hidden_dims": null,
|
| 816 |
+
"edit_hidden_dims": [
|
| 817 |
+
256,
|
| 818 |
+
256,
|
| 819 |
+
256
|
| 820 |
+
],
|
| 821 |
+
"edit_init_temperature": 1.0,
|
| 822 |
+
"edit_layer_norm": true,
|
| 823 |
+
"edit_lr": 0.0003,
|
| 824 |
+
"edit_policy_bound": 0.1,
|
| 825 |
+
"edit_start_step": 100000,
|
| 826 |
+
"edit_target_entropy": null,
|
| 827 |
+
"edit_weight_decay": 0.0,
|
| 828 |
+
"encoder": null,
|
| 829 |
+
"entropy_backup": false,
|
| 830 |
+
"flow_steps": 10,
|
| 831 |
+
"fql_distillation_coeff": 10.0,
|
| 832 |
+
"fql_train_main_policy": true,
|
| 833 |
+
"freeze_vision_encoder": false,
|
| 834 |
+
"ft_flow_steps": 10,
|
| 835 |
+
"horizon_length": 8,
|
| 836 |
+
"init_temperature": 1.0,
|
| 837 |
+
"layer_norm": true,
|
| 838 |
+
"lr": 0.0003,
|
| 839 |
+
"lr_action_critic": null,
|
| 840 |
+
"lr_actor": null,
|
| 841 |
+
"lr_actor_bc_flow": null,
|
| 842 |
+
"lr_critic": null,
|
| 843 |
+
"lr_edit_actor": null,
|
| 844 |
+
"lr_noise_critic": null,
|
| 845 |
+
"max_noise_std": 0.01,
|
| 846 |
+
"mc_regression": false,
|
| 847 |
+
"mc_regression_coeff": 1.0,
|
| 848 |
+
"min_noise_std": 0.01,
|
| 849 |
+
"mip_q_ensemble": false,
|
| 850 |
+
"mip_q_noise_scale": 1.0,
|
| 851 |
+
"mip_q_t_star": 0.9,
|
| 852 |
+
"mip_t_star": 0.999,
|
| 853 |
+
"muon_beta": 0.95,
|
| 854 |
+
"muon_nesterov": true,
|
| 855 |
+
"muon_ns_steps": 5,
|
| 856 |
+
"n_action_samples": 8,
|
| 857 |
+
"n_edit_samples": 8,
|
| 858 |
+
"noise_critic_grad_steps": 1,
|
| 859 |
+
"noise_dim": null,
|
| 860 |
+
"noise_net_hidden_dims": [
|
| 861 |
+
256,
|
| 862 |
+
256
|
| 863 |
+
],
|
| 864 |
+
"noise_net_layer_norm": true,
|
| 865 |
+
"normalize_act_space_dimension": false,
|
| 866 |
+
"normalize_denoising_horizon": false,
|
| 867 |
+
"normalize_q_loss": false,
|
| 868 |
+
"num_bins": 256,
|
| 869 |
+
"num_ensemble_members": 10,
|
| 870 |
+
"num_qs": 10,
|
| 871 |
+
"ob_dims": null,
|
| 872 |
+
"one_step_decay_steps": null,
|
| 873 |
+
"one_step_end_value": null,
|
| 874 |
+
"one_step_hidden_dims": [
|
| 875 |
+
512,
|
| 876 |
+
512,
|
| 877 |
+
512,
|
| 878 |
+
512
|
| 879 |
+
],
|
| 880 |
+
"one_step_layer_norm": false,
|
| 881 |
+
"one_step_lr": null,
|
| 882 |
+
"one_step_scheduler": null,
|
| 883 |
+
"one_step_warmup_steps": null,
|
| 884 |
+
"optimizer_action_critic": null,
|
| 885 |
+
"optimizer_actor": "adam",
|
| 886 |
+
"optimizer_actor_bc_flow": "adam",
|
| 887 |
+
"optimizer_critic": "adam",
|
| 888 |
+
"optimizer_edit_actor": null,
|
| 889 |
+
"paligemma_tokenizer_path": null,
|
| 890 |
+
"partition_optimizer": true,
|
| 891 |
+
"policy_type": "flow",
|
| 892 |
+
"ppo_batch_size": 256,
|
| 893 |
+
"ppo_lr": 4.5e-05,
|
| 894 |
+
"proprio_dim": null,
|
| 895 |
+
"q_agg": "mean",
|
| 896 |
+
"q_max": null,
|
| 897 |
+
"q_min": null,
|
| 898 |
+
"q_target_agg": "subset_min",
|
| 899 |
+
"q_target_replace": false,
|
| 900 |
+
"q_target_subset": 2,
|
| 901 |
+
"q_variance_reduction": false,
|
| 902 |
+
"q_vr_num_samples": 8,
|
| 903 |
+
"q_vr_reduction": "mean",
|
| 904 |
+
"randn_clip_value": 3.0,
|
| 905 |
+
"shortcut_denoising_steps": 4,
|
| 906 |
+
"shortcut_inference_steps": 4,
|
| 907 |
+
"simba_actor_hidden_dims": [
|
| 908 |
+
1024,
|
| 909 |
+
1024
|
| 910 |
+
],
|
| 911 |
+
"simba_critic_hidden_dims": [
|
| 912 |
+
1024,
|
| 913 |
+
1024,
|
| 914 |
+
1024
|
| 915 |
+
],
|
| 916 |
+
"subsample_bon": false,
|
| 917 |
+
"target_entropy": null,
|
| 918 |
+
"tau": 0.005,
|
| 919 |
+
"tf_pi_conditioning": "adaln",
|
| 920 |
+
"tf_pi_dropout": 0.0,
|
| 921 |
+
"tf_pi_embed_dim": 256,
|
| 922 |
+
"tf_pi_heads": 4,
|
| 923 |
+
"tf_pi_layers": 8,
|
| 924 |
+
"tf_q_conditioning": "adaln",
|
| 925 |
+
"tf_q_dropout": 0.0,
|
| 926 |
+
"tf_q_embed_dim": 256,
|
| 927 |
+
"tf_q_heads": 4,
|
| 928 |
+
"tf_q_layers": 8,
|
| 929 |
+
"time_embedding": "sinusoidal",
|
| 930 |
+
"time_embedding_dim": 32,
|
| 931 |
+
"train_one_step_in_offline": true,
|
| 932 |
+
"train_one_step_in_online": true,
|
| 933 |
+
"useFiLM": false,
|
| 934 |
+
"useSimBa": false,
|
| 935 |
+
"useTF": false,
|
| 936 |
+
"use_constant_noise": true,
|
| 937 |
+
"use_constant_scheduler_for_bc": true,
|
| 938 |
+
"use_denoiser": false,
|
| 939 |
+
"use_edit_actor": false,
|
| 940 |
+
"use_mip_q": false,
|
| 941 |
+
"use_muon": false,
|
| 942 |
+
"use_one_step_policy": false,
|
| 943 |
+
"use_shortcut": false,
|
| 944 |
+
"use_state": "proprio",
|
| 945 |
+
"use_success_buffer_q": false,
|
| 946 |
+
"value_hidden_dims": [
|
| 947 |
+
512,
|
| 948 |
+
512,
|
| 949 |
+
512
|
| 950 |
+
],
|
| 951 |
+
"vit_encoder": "minvit",
|
| 952 |
+
"vit_img_proj_dim": 0,
|
| 953 |
+
"vit_state_proj_dim": 0
|
| 954 |
+
},
|
| 955 |
+
"bc": {
|
| 956 |
+
"clip_bc": true,
|
| 957 |
+
"clip_bc_threshold": 0.45,
|
| 958 |
+
"clip_bc_wrt": "ODE"
|
| 959 |
+
},
|
| 960 |
+
"buffer": {
|
| 961 |
+
"offline_ratio": 0.0,
|
| 962 |
+
"size": 2000000,
|
| 963 |
+
"start_training": 10000,
|
| 964 |
+
"utd_online": 1,
|
| 965 |
+
"utd_pi": null,
|
| 966 |
+
"utd_q": null,
|
| 967 |
+
"utd_v": null,
|
| 968 |
+
"utd_warmup": 1
|
| 969 |
+
},
|
| 970 |
+
"checkpoint": {
|
| 971 |
+
"ep_resume": 0,
|
| 972 |
+
"restore_actor_path": null,
|
| 973 |
+
"restore_critic_path": null,
|
| 974 |
+
"restore_encoder_path": null
|
| 975 |
+
},
|
| 976 |
+
"dataset": {
|
| 977 |
+
"num_offline_trajs": -1,
|
| 978 |
+
"ogbench_dir": null,
|
| 979 |
+
"p_aug": 0.0,
|
| 980 |
+
"proportion": 1.0,
|
| 981 |
+
"replace_interval": 1000
|
| 982 |
+
},
|
| 983 |
+
"env": {
|
| 984 |
+
"name": "tool_hang-ph-low_dim",
|
| 985 |
+
"post_success_steps": 0,
|
| 986 |
+
"rew_fn": "sparse",
|
| 987 |
+
"sparse": false
|
| 988 |
+
},
|
| 989 |
+
"eval": {
|
| 990 |
+
"eval_episodes": 2,
|
| 991 |
+
"n_eval_envs": 32,
|
| 992 |
+
"plot_q_vs_mc": false,
|
| 993 |
+
"video_episodes": 0,
|
| 994 |
+
"video_frame_skip": 3
|
| 995 |
+
},
|
| 996 |
+
"experiment": {
|
| 997 |
+
"log": true,
|
| 998 |
+
"project": "QC",
|
| 999 |
+
"run_group": "toolhang",
|
| 1000 |
+
"run_name": "qc_toolhang",
|
| 1001 |
+
"save_dir": "exp/ogpo",
|
| 1002 |
+
"seed": 1,
|
| 1003 |
+
"wandb_name": "qc_toolhang"
|
| 1004 |
+
},
|
| 1005 |
+
"intervals": {
|
| 1006 |
+
"eval_interval": 20000,
|
| 1007 |
+
"eval_interval_bc": 5000,
|
| 1008 |
+
"log_interval": 5000,
|
| 1009 |
+
"preemption_backup_interval": -1,
|
| 1010 |
+
"save_interval": 500000,
|
| 1011 |
+
"save_interval_bc": -1,
|
| 1012 |
+
"step_data_log_interval": -1
|
| 1013 |
+
},
|
| 1014 |
+
"success_buffer": {
|
| 1015 |
+
"batch_size": 256,
|
| 1016 |
+
"use_success_buffer": false
|
| 1017 |
+
},
|
| 1018 |
+
"training": {
|
| 1019 |
+
"bc_only": false,
|
| 1020 |
+
"bc_pi_steps": 0,
|
| 1021 |
+
"bc_q_steps": 0,
|
| 1022 |
+
"bc_refine_steps": 0,
|
| 1023 |
+
"calql_steps": 0,
|
| 1024 |
+
"force_jax_sync": true,
|
| 1025 |
+
"gpu_mode": "single",
|
| 1026 |
+
"online_steps": 5000000,
|
| 1027 |
+
"q_warmup_steps": 0
|
| 1028 |
+
}
|
| 1029 |
+
},
|
| 1030 |
+
"upstream_source": {
|
| 1031 |
+
"ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
|
| 1032 |
+
"patches_sha256": {
|
| 1033 |
+
"ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
|
| 1034 |
+
"ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
|
| 1035 |
+
"ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
|
| 1036 |
+
"ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
|
| 1037 |
+
"ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
|
| 1038 |
+
}
|
| 1039 |
+
}
|
| 1040 |
+
},
|
| 1041 |
+
"observation_mode": "pixel_state",
|
| 1042 |
+
"offline_buffer": {
|
| 1043 |
+
"actions_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1044 |
+
"aux_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1045 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 1046 |
+
"chunks": 6171,
|
| 1047 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1048 |
+
"decision_ref": "[1017-S2OFFBC]",
|
| 1049 |
+
"env_steps": 48984,
|
| 1050 |
+
"features_sha256": "fb059cee8121cd0039a4ac218f1a19de767b07cf81010471e4342480fc9119fa",
|
| 1051 |
+
"horizon": 8,
|
| 1052 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 1053 |
+
"learner_coordinates": "JaxStudy2Learner:qc",
|
| 1054 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1055 |
+
"ratio": {
|
| 1056 |
+
"adaptive_ratio": false,
|
| 1057 |
+
"rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
|
| 1058 |
+
},
|
| 1059 |
+
"selection": {
|
| 1060 |
+
"cohort_size": 100,
|
| 1061 |
+
"demos": [
|
| 1062 |
+
"demo_0",
|
| 1063 |
+
"demo_2",
|
| 1064 |
+
"demo_5",
|
| 1065 |
+
"demo_7",
|
| 1066 |
+
"demo_9",
|
| 1067 |
+
"demo_10",
|
| 1068 |
+
"demo_13",
|
| 1069 |
+
"demo_16",
|
| 1070 |
+
"demo_17",
|
| 1071 |
+
"demo_18",
|
| 1072 |
+
"demo_20",
|
| 1073 |
+
"demo_23",
|
| 1074 |
+
"demo_27",
|
| 1075 |
+
"demo_28",
|
| 1076 |
+
"demo_29",
|
| 1077 |
+
"demo_30",
|
| 1078 |
+
"demo_31",
|
| 1079 |
+
"demo_32",
|
| 1080 |
+
"demo_34",
|
| 1081 |
+
"demo_35",
|
| 1082 |
+
"demo_37",
|
| 1083 |
+
"demo_40",
|
| 1084 |
+
"demo_41",
|
| 1085 |
+
"demo_43",
|
| 1086 |
+
"demo_46",
|
| 1087 |
+
"demo_47",
|
| 1088 |
+
"demo_50",
|
| 1089 |
+
"demo_51",
|
| 1090 |
+
"demo_56",
|
| 1091 |
+
"demo_57",
|
| 1092 |
+
"demo_58",
|
| 1093 |
+
"demo_67",
|
| 1094 |
+
"demo_71",
|
| 1095 |
+
"demo_74",
|
| 1096 |
+
"demo_76",
|
| 1097 |
+
"demo_78",
|
| 1098 |
+
"demo_80",
|
| 1099 |
+
"demo_82",
|
| 1100 |
+
"demo_83",
|
| 1101 |
+
"demo_85",
|
| 1102 |
+
"demo_86",
|
| 1103 |
+
"demo_87",
|
| 1104 |
+
"demo_93",
|
| 1105 |
+
"demo_97",
|
| 1106 |
+
"demo_98",
|
| 1107 |
+
"demo_101",
|
| 1108 |
+
"demo_102",
|
| 1109 |
+
"demo_107",
|
| 1110 |
+
"demo_110",
|
| 1111 |
+
"demo_111",
|
| 1112 |
+
"demo_112",
|
| 1113 |
+
"demo_113",
|
| 1114 |
+
"demo_115",
|
| 1115 |
+
"demo_116",
|
| 1116 |
+
"demo_119",
|
| 1117 |
+
"demo_122",
|
| 1118 |
+
"demo_123",
|
| 1119 |
+
"demo_126",
|
| 1120 |
+
"demo_127",
|
| 1121 |
+
"demo_128",
|
| 1122 |
+
"demo_130",
|
| 1123 |
+
"demo_131",
|
| 1124 |
+
"demo_132",
|
| 1125 |
+
"demo_133",
|
| 1126 |
+
"demo_134",
|
| 1127 |
+
"demo_135",
|
| 1128 |
+
"demo_138",
|
| 1129 |
+
"demo_140",
|
| 1130 |
+
"demo_141",
|
| 1131 |
+
"demo_145",
|
| 1132 |
+
"demo_147",
|
| 1133 |
+
"demo_149",
|
| 1134 |
+
"demo_151",
|
| 1135 |
+
"demo_152",
|
| 1136 |
+
"demo_154",
|
| 1137 |
+
"demo_156",
|
| 1138 |
+
"demo_157",
|
| 1139 |
+
"demo_159",
|
| 1140 |
+
"demo_161",
|
| 1141 |
+
"demo_163",
|
| 1142 |
+
"demo_164",
|
| 1143 |
+
"demo_171",
|
| 1144 |
+
"demo_172",
|
| 1145 |
+
"demo_173",
|
| 1146 |
+
"demo_174",
|
| 1147 |
+
"demo_175",
|
| 1148 |
+
"demo_176",
|
| 1149 |
+
"demo_177",
|
| 1150 |
+
"demo_179",
|
| 1151 |
+
"demo_181",
|
| 1152 |
+
"demo_182",
|
| 1153 |
+
"demo_183",
|
| 1154 |
+
"demo_186",
|
| 1155 |
+
"demo_188",
|
| 1156 |
+
"demo_190",
|
| 1157 |
+
"demo_191",
|
| 1158 |
+
"demo_192",
|
| 1159 |
+
"demo_193",
|
| 1160 |
+
"demo_196",
|
| 1161 |
+
"demo_199"
|
| 1162 |
+
],
|
| 1163 |
+
"manifest": "/ssd/WorkSpace/RL-Postrain/checkpoints/huggingface/yangfy0627/RL-postTrain/e62671ed1cf646b453660b698c22ca4d273b5ccc/Robomimic/study1/ToolHang/diverse100/seed_1/data_manifest.json",
|
| 1164 |
+
"manifest_sha256": "9233d5824205e29f1afa721d8573fe19805a1a8e7829c296af1dd0b74a1d29b4",
|
| 1165 |
+
"regime": "diverse100",
|
| 1166 |
+
"source": "bc_regime",
|
| 1167 |
+
"source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1168 |
+
"split": {
|
| 1169 |
+
"training": 90,
|
| 1170 |
+
"validation": 10
|
| 1171 |
+
},
|
| 1172 |
+
"task": "tool_hang"
|
| 1173 |
+
}
|
| 1174 |
+
},
|
| 1175 |
+
"offline_role": "mixed",
|
| 1176 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1177 |
+
"smoke": false,
|
| 1178 |
+
"task": "tool_hang",
|
| 1179 |
+
"task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
|
| 1180 |
+
},
|
| 1181 |
+
"seed": 1,
|
| 1182 |
+
"start_policy": "base_cfm_ode",
|
| 1183 |
+
"variant": {
|
| 1184 |
+
"flags": {
|
| 1185 |
+
"bon_sft": false,
|
| 1186 |
+
"use_success_buffer": false
|
| 1187 |
+
},
|
| 1188 |
+
"learner": "qc",
|
| 1189 |
+
"name": "qc",
|
| 1190 |
+
"phase": 1,
|
| 1191 |
+
"spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
|
| 1192 |
+
"version": "study2_variants_v4"
|
| 1193 |
+
},
|
| 1194 |
+
"variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
|
| 1195 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/best/state.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc32c5a044147dca30b67057fc8d732e1a35ad69786d41a690ce1977cff0bb57
|
| 3 |
+
size 207200290
|
RL/ToolHang100/QC/diverse100/seed_1/comparison_evaluations.jsonl
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"eval_index": 0, "rl_env_steps": 0, "target_step": 0, "sr": 0.19, "round": 0, "update_index": 0, "policy_version": 0, "chunk_transitions": 0, "gradient_updates": 0, "n_success": 38, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 2 |
+
{"eval_index": 1, "rl_env_steps": 23971, "target_step": 20000, "sr": 0.245, "round": 3, "update_index": 9000, "policy_version": 3, "chunk_transitions": 3000, "gradient_updates": 9000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 3 |
+
{"eval_index": 2, "rl_env_steps": 47944, "target_step": 40000, "sr": 0.22, "round": 6, "update_index": 18000, "policy_version": 6, "chunk_transitions": 6000, "gradient_updates": 18000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 4 |
+
{"eval_index": 3, "rl_env_steps": 63925, "target_step": 60000, "sr": 0.27, "round": 8, "update_index": 24000, "policy_version": 8, "chunk_transitions": 8000, "gradient_updates": 24000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 5 |
+
{"eval_index": 4, "rl_env_steps": 87901, "target_step": 80000, "sr": 0.335, "round": 11, "update_index": 33000, "policy_version": 11, "chunk_transitions": 11000, "gradient_updates": 33000, "n_success": 67, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 6 |
+
{"eval_index": 5, "rl_env_steps": 103895, "target_step": 100000, "sr": 0.26, "round": 13, "update_index": 39000, "policy_version": 13, "chunk_transitions": 13000, "gradient_updates": 39000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 7 |
+
{"eval_index": 6, "rl_env_steps": 127876, "target_step": 120000, "sr": 0.16, "round": 16, "update_index": 48000, "policy_version": 16, "chunk_transitions": 16000, "gradient_updates": 48000, "n_success": 32, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 8 |
+
{"eval_index": 7, "rl_env_steps": 143854, "target_step": 140000, "sr": 0.25, "round": 18, "update_index": 54000, "policy_version": 18, "chunk_transitions": 18000, "gradient_updates": 54000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 9 |
+
{"eval_index": 8, "rl_env_steps": 167826, "target_step": 160000, "sr": 0.295, "round": 21, "update_index": 63000, "policy_version": 21, "chunk_transitions": 21000, "gradient_updates": 63000, "n_success": 59, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": null, "gain": null, "observed": true, "imputed": false}
|
| 10 |
+
{"eval_index": 9, "rl_env_steps": 183814, "target_step": 180000, "sr": 0.265, "round": 23, "update_index": 69000, "policy_version": 23, "chunk_transitions": 23000, "gradient_updates": 69000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24900000000000003, "gain": null, "observed": true, "imputed": false}
|
| 11 |
+
{"eval_index": 10, "rl_env_steps": 207787, "target_step": 200000, "sr": 0.215, "round": 26, "update_index": 78000, "policy_version": 26, "chunk_transitions": 26000, "gradient_updates": 78000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25149999999999995, "gain": null, "observed": true, "imputed": false}
|
| 12 |
+
{"eval_index": 11, "rl_env_steps": 223769, "target_step": 220000, "sr": 0.23, "round": 28, "update_index": 84000, "policy_version": 28, "chunk_transitions": 28000, "gradient_updates": 84000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25, "gain": null, "observed": true, "imputed": false}
|
| 13 |
+
{"eval_index": 12, "rl_env_steps": 247753, "target_step": 240000, "sr": 0.195, "round": 31, "update_index": 93000, "policy_version": 31, "chunk_transitions": 31000, "gradient_updates": 93000, "n_success": 39, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24749999999999997, "gain": null, "observed": true, "imputed": false}
|
| 14 |
+
{"eval_index": 13, "rl_env_steps": 263739, "target_step": 260000, "sr": 0.25, "round": 33, "update_index": 99000, "policy_version": 33, "chunk_transitions": 33000, "gradient_updates": 99000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24549999999999997, "gain": null, "observed": true, "imputed": false}
|
| 15 |
+
{"eval_index": 14, "rl_env_steps": 287729, "target_step": 280000, "sr": 0.25, "round": 36, "update_index": 108000, "policy_version": 36, "chunk_transitions": 36000, "gradient_updates": 108000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23700000000000002, "gain": null, "observed": true, "imputed": false}
|
| 16 |
+
{"eval_index": 15, "rl_env_steps": 303714, "target_step": 300000, "sr": 0.205, "round": 38, "update_index": 114000, "policy_version": 38, "chunk_transitions": 38000, "gradient_updates": 114000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23150000000000004, "gain": null, "observed": true, "imputed": false}
|
| 17 |
+
{"eval_index": 16, "rl_env_steps": 327675, "target_step": 320000, "sr": 0.27, "round": 41, "update_index": 123000, "policy_version": 41, "chunk_transitions": 41000, "gradient_updates": 123000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2425, "gain": null, "observed": true, "imputed": false}
|
| 18 |
+
{"eval_index": 17, "rl_env_steps": 343660, "target_step": 340000, "sr": 0.2, "round": 43, "update_index": 129000, "policy_version": 43, "chunk_transitions": 43000, "gradient_updates": 129000, "n_success": 40, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2375, "gain": null, "observed": true, "imputed": false}
|
| 19 |
+
{"eval_index": 18, "rl_env_steps": 367622, "target_step": 360000, "sr": 0.26, "round": 46, "update_index": 138000, "policy_version": 46, "chunk_transitions": 46000, "gradient_updates": 138000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23399999999999999, "gain": null, "observed": true, "imputed": false}
|
| 20 |
+
{"eval_index": 19, "rl_env_steps": 383610, "target_step": 380000, "sr": 0.24, "round": 48, "update_index": 144000, "policy_version": 48, "chunk_transitions": 48000, "gradient_updates": 144000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23150000000000004, "gain": -0.017499999999999988, "observed": true, "imputed": false}
|
| 21 |
+
{"eval_index": 20, "rl_env_steps": 407593, "target_step": 400000, "sr": 0.21, "round": 51, "update_index": 153000, "policy_version": 51, "chunk_transitions": 51000, "gradient_updates": 153000, "n_success": 42, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.231, "gain": -0.020499999999999935, "observed": true, "imputed": false}
|
| 22 |
+
{"eval_index": 21, "rl_env_steps": 423591, "target_step": 420000, "sr": 0.32, "round": 53, "update_index": 159000, "policy_version": 53, "chunk_transitions": 53000, "gradient_updates": 159000, "n_success": 64, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24, "gain": -0.010000000000000009, "observed": true, "imputed": false}
|
| 23 |
+
{"eval_index": 22, "rl_env_steps": 447565, "target_step": 440000, "sr": 0.255, "round": 56, "update_index": 168000, "policy_version": 56, "chunk_transitions": 56000, "gradient_updates": 168000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24599999999999994, "gain": -0.001500000000000029, "observed": true, "imputed": false}
|
| 24 |
+
{"eval_index": 23, "rl_env_steps": 463565, "target_step": 460000, "sr": 0.26, "round": 58, "update_index": 174000, "policy_version": 58, "chunk_transitions": 58000, "gradient_updates": 174000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24699999999999997, "gain": 0.0015000000000000013, "observed": true, "imputed": false}
|
| 25 |
+
{"eval_index": 24, "rl_env_steps": 487539, "target_step": 480000, "sr": 0.28, "round": 61, "update_index": 183000, "policy_version": 61, "chunk_transitions": 61000, "gradient_updates": 183000, "n_success": 56, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25, "gain": 0.012999999999999984, "observed": true, "imputed": false}
|
| 26 |
+
{"eval_index": 25, "rl_env_steps": 503527, "target_step": 500000, "sr": 0.255, "round": 63, "update_index": 189000, "policy_version": 63, "chunk_transitions": 63000, "gradient_updates": 189000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.255, "gain": 0.023499999999999965, "observed": true, "imputed": false}
|
| 27 |
+
{"eval_index": 26, "rl_env_steps": 527509, "target_step": 520000, "sr": 0.205, "round": 66, "update_index": 198000, "policy_version": 66, "chunk_transitions": 66000, "gradient_updates": 198000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2485, "gain": 0.006000000000000005, "observed": true, "imputed": false}
|
| 28 |
+
{"eval_index": 27, "rl_env_steps": 543493, "target_step": 540000, "sr": 0.295, "round": 68, "update_index": 204000, "policy_version": 68, "chunk_transitions": 68000, "gradient_updates": 204000, "n_success": 59, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.258, "gain": 0.020500000000000018, "observed": true, "imputed": false}
|
| 29 |
+
{"eval_index": 28, "rl_env_steps": 567458, "target_step": 560000, "sr": 0.23, "round": 71, "update_index": 213000, "policy_version": 71, "chunk_transitions": 71000, "gradient_updates": 213000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.255, "gain": 0.02100000000000002, "observed": true, "imputed": false}
|
| 30 |
+
{"eval_index": 29, "rl_env_steps": 583442, "target_step": 580000, "sr": 0.25, "round": 73, "update_index": 219000, "policy_version": 73, "chunk_transitions": 73000, "gradient_updates": 219000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.256, "gain": 0.024499999999999966, "observed": true, "imputed": false}
|
| 31 |
+
{"eval_index": 30, "rl_env_steps": 607421, "target_step": 600000, "sr": 0.215, "round": 76, "update_index": 228000, "policy_version": 76, "chunk_transitions": 76000, "gradient_updates": 228000, "n_success": 43, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2565, "gain": 0.025499999999999995, "observed": true, "imputed": false}
|
| 32 |
+
{"eval_index": 31, "rl_env_steps": 623412, "target_step": 620000, "sr": 0.29, "round": 78, "update_index": 234000, "policy_version": 78, "chunk_transitions": 78000, "gradient_updates": 234000, "n_success": 58, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2535, "gain": 0.013500000000000012, "observed": true, "imputed": false}
|
| 33 |
+
{"eval_index": 32, "rl_env_steps": 647389, "target_step": 640000, "sr": 0.165, "round": 81, "update_index": 243000, "policy_version": 81, "chunk_transitions": 81000, "gradient_updates": 243000, "n_success": 33, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2445, "gain": -0.0014999999999999458, "observed": true, "imputed": false}
|
| 34 |
+
{"eval_index": 33, "rl_env_steps": 663375, "target_step": 660000, "sr": 0.265, "round": 83, "update_index": 249000, "policy_version": 83, "chunk_transitions": 83000, "gradient_updates": 249000, "n_success": 53, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24500000000000002, "gain": -0.0019999999999999463, "observed": true, "imputed": false}
|
| 35 |
+
{"eval_index": 34, "rl_env_steps": 687326, "target_step": 680000, "sr": 0.2, "round": 86, "update_index": 258000, "policy_version": 86, "chunk_transitions": 86000, "gradient_updates": 258000, "n_success": 40, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23700000000000002, "gain": -0.012999999999999984, "observed": true, "imputed": false}
|
| 36 |
+
{"eval_index": 35, "rl_env_steps": 703314, "target_step": 700000, "sr": 0.25, "round": 88, "update_index": 264000, "policy_version": 88, "chunk_transitions": 88000, "gradient_updates": 264000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23650000000000002, "gain": -0.01849999999999999, "observed": true, "imputed": false}
|
| 37 |
+
{"eval_index": 36, "rl_env_steps": 727280, "target_step": 720000, "sr": 0.255, "round": 91, "update_index": 273000, "policy_version": 91, "chunk_transitions": 91000, "gradient_updates": 273000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2415, "gain": -0.007000000000000006, "observed": true, "imputed": false}
|
| 38 |
+
{"eval_index": 37, "rl_env_steps": 743258, "target_step": 740000, "sr": 0.285, "round": 93, "update_index": 279000, "policy_version": 93, "chunk_transitions": 93000, "gradient_updates": 279000, "n_success": 57, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24050000000000002, "gain": -0.017499999999999988, "observed": true, "imputed": false}
|
| 39 |
+
{"eval_index": 38, "rl_env_steps": 767232, "target_step": 760000, "sr": 0.19, "round": 96, "update_index": 288000, "policy_version": 96, "chunk_transitions": 96000, "gradient_updates": 288000, "n_success": 38, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2365, "gain": -0.018500000000000016, "observed": true, "imputed": false}
|
| 40 |
+
{"eval_index": 39, "rl_env_steps": 783208, "target_step": 780000, "sr": 0.27, "round": 98, "update_index": 294000, "policy_version": 98, "chunk_transitions": 98000, "gradient_updates": 294000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23850000000000002, "gain": -0.017499999999999988, "observed": true, "imputed": false}
|
| 41 |
+
{"eval_index": 40, "rl_env_steps": 807164, "target_step": 800000, "sr": 0.245, "round": 101, "update_index": 303000, "policy_version": 101, "chunk_transitions": 101000, "gradient_updates": 303000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2415, "gain": -0.015000000000000013, "observed": true, "imputed": false}
|
| 42 |
+
{"eval_index": 41, "rl_env_steps": 823128, "target_step": 820000, "sr": 0.22, "round": 103, "update_index": 309000, "policy_version": 103, "chunk_transitions": 103000, "gradient_updates": 309000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23450000000000001, "gain": -0.01899999999999999, "observed": true, "imputed": false}
|
| 43 |
+
{"eval_index": 42, "rl_env_steps": 847114, "target_step": 840000, "sr": 0.2, "round": 106, "update_index": 318000, "policy_version": 106, "chunk_transitions": 106000, "gradient_updates": 318000, "n_success": 40, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23800000000000004, "gain": -0.00649999999999995, "observed": true, "imputed": false}
|
| 44 |
+
{"eval_index": 43, "rl_env_steps": 863099, "target_step": 860000, "sr": 0.29, "round": 108, "update_index": 324000, "policy_version": 108, "chunk_transitions": 108000, "gradient_updates": 324000, "n_success": 58, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24050000000000002, "gain": -0.004500000000000004, "observed": true, "imputed": false}
|
| 45 |
+
{"eval_index": 44, "rl_env_steps": 887074, "target_step": 880000, "sr": 0.27, "round": 111, "update_index": 333000, "policy_version": 111, "chunk_transitions": 111000, "gradient_updates": 333000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2475, "gain": 0.010499999999999982, "observed": true, "imputed": false}
|
| 46 |
+
{"eval_index": 45, "rl_env_steps": 903052, "target_step": 900000, "sr": 0.165, "round": 113, "update_index": 339000, "policy_version": 113, "chunk_transitions": 113000, "gradient_updates": 339000, "n_success": 33, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23900000000000002, "gain": 0.0025000000000000022, "observed": true, "imputed": false}
|
| 47 |
+
{"eval_index": 46, "rl_env_steps": 927021, "target_step": 920000, "sr": 0.195, "round": 116, "update_index": 348000, "policy_version": 116, "chunk_transitions": 116000, "gradient_updates": 348000, "n_success": 39, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.233, "gain": -0.00849999999999998, "observed": true, "imputed": false}
|
| 48 |
+
{"eval_index": 47, "rl_env_steps": 943005, "target_step": 940000, "sr": 0.185, "round": 118, "update_index": 354000, "policy_version": 118, "chunk_transitions": 118000, "gradient_updates": 354000, "n_success": 37, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.223, "gain": -0.017500000000000016, "observed": true, "imputed": false}
|
| 49 |
+
{"eval_index": 48, "rl_env_steps": 966968, "target_step": 960000, "sr": 0.26, "round": 121, "update_index": 363000, "policy_version": 121, "chunk_transitions": 121000, "gradient_updates": 363000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22999999999999998, "gain": -0.006500000000000006, "observed": true, "imputed": false}
|
| 50 |
+
{"eval_index": 49, "rl_env_steps": 982951, "target_step": 980000, "sr": 0.27, "round": 123, "update_index": 369000, "policy_version": 123, "chunk_transitions": 123000, "gradient_updates": 369000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23000000000000004, "gain": -0.00849999999999998, "observed": true, "imputed": false}
|
| 51 |
+
{"eval_index": 50, "rl_env_steps": 1006914, "target_step": 1000000, "sr": 0.245, "round": 126, "update_index": 378000, "policy_version": 126, "chunk_transitions": 126000, "gradient_updates": 378000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23000000000000004, "gain": -0.011499999999999955, "observed": true, "imputed": false}
|
| 52 |
+
{"eval_index": 51, "rl_env_steps": 1022908, "target_step": 1020000, "sr": 0.255, "round": 128, "update_index": 384000, "policy_version": 128, "chunk_transitions": 128000, "gradient_updates": 384000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23349999999999999, "gain": -0.0010000000000000286, "observed": true, "imputed": false}
|
| 53 |
+
{"eval_index": 52, "rl_env_steps": 1046888, "target_step": 1040000, "sr": 0.27, "round": 131, "update_index": 393000, "policy_version": 131, "chunk_transitions": 131000, "gradient_updates": 393000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2405, "gain": 0.0024999999999999467, "observed": true, "imputed": false}
|
| 54 |
+
{"eval_index": 53, "rl_env_steps": 1062868, "target_step": 1060000, "sr": 0.28, "round": 133, "update_index": 399000, "policy_version": 133, "chunk_transitions": 133000, "gradient_updates": 399000, "n_success": 56, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23950000000000005, "gain": -0.0009999999999999731, "observed": true, "imputed": false}
|
| 55 |
+
{"eval_index": 54, "rl_env_steps": 1086846, "target_step": 1080000, "sr": 0.24, "round": 136, "update_index": 408000, "policy_version": 136, "chunk_transitions": 136000, "gradient_updates": 408000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23650000000000002, "gain": -0.010999999999999982, "observed": true, "imputed": false}
|
| 56 |
+
{"eval_index": 55, "rl_env_steps": 1102833, "target_step": 1100000, "sr": 0.29, "round": 138, "update_index": 414000, "policy_version": 138, "chunk_transitions": 138000, "gradient_updates": 414000, "n_success": 58, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24900000000000003, "gain": 0.010000000000000009, "observed": true, "imputed": false}
|
| 57 |
+
{"eval_index": 56, "rl_env_steps": 1126809, "target_step": 1120000, "sr": 0.21, "round": 141, "update_index": 423000, "policy_version": 141, "chunk_transitions": 141000, "gradient_updates": 423000, "n_success": 42, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2505, "gain": 0.017499999999999988, "observed": true, "imputed": false}
|
| 58 |
+
{"eval_index": 57, "rl_env_steps": 1142797, "target_step": 1140000, "sr": 0.245, "round": 143, "update_index": 429000, "policy_version": 143, "chunk_transitions": 143000, "gradient_updates": 429000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.25650000000000006, "gain": 0.03350000000000006, "observed": true, "imputed": false}
|
| 59 |
+
{"eval_index": 58, "rl_env_steps": 1166783, "target_step": 1160000, "sr": 0.245, "round": 146, "update_index": 438000, "policy_version": 146, "chunk_transitions": 146000, "gradient_updates": 438000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.255, "gain": 0.025000000000000022, "observed": true, "imputed": false}
|
| 60 |
+
{"eval_index": 59, "rl_env_steps": 1182767, "target_step": 1180000, "sr": 0.23, "round": 148, "update_index": 444000, "policy_version": 148, "chunk_transitions": 148000, "gradient_updates": 444000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.251, "gain": 0.020999999999999963, "observed": true, "imputed": false}
|
| 61 |
+
{"eval_index": 60, "rl_env_steps": 1206758, "target_step": 1200000, "sr": 0.22, "round": 151, "update_index": 453000, "policy_version": 151, "chunk_transitions": 151000, "gradient_updates": 453000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.24850000000000003, "gain": 0.01849999999999999, "observed": true, "imputed": false}
|
| 62 |
+
{"eval_index": 61, "rl_env_steps": 1222750, "target_step": 1220000, "sr": 0.205, "round": 153, "update_index": 459000, "policy_version": 153, "chunk_transitions": 153000, "gradient_updates": 459000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2435, "gain": 0.010000000000000009, "observed": true, "imputed": false}
|
| 63 |
+
{"eval_index": 62, "rl_env_steps": 1246744, "target_step": 1240000, "sr": 0.26, "round": 156, "update_index": 468000, "policy_version": 156, "chunk_transitions": 156000, "gradient_updates": 468000, "n_success": 52, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2425, "gain": 0.0020000000000000018, "observed": true, "imputed": false}
|
| 64 |
+
{"eval_index": 63, "rl_env_steps": 1262713, "target_step": 1260000, "sr": 0.24, "round": 158, "update_index": 474000, "policy_version": 158, "chunk_transitions": 158000, "gradient_updates": 474000, "n_success": 48, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2385, "gain": -0.0010000000000000564, "observed": true, "imputed": false}
|
| 65 |
+
{"eval_index": 64, "rl_env_steps": 1286701, "target_step": 1280000, "sr": 0.205, "round": 161, "update_index": 483000, "policy_version": 161, "chunk_transitions": 161000, "gradient_updates": 483000, "n_success": 41, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23500000000000001, "gain": -0.0015000000000000013, "observed": true, "imputed": false}
|
| 66 |
+
{"eval_index": 65, "rl_env_steps": 1302679, "target_step": 1300000, "sr": 0.245, "round": 163, "update_index": 489000, "policy_version": 163, "chunk_transitions": 163000, "gradient_updates": 489000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2305, "gain": -0.018500000000000016, "observed": true, "imputed": false}
|
| 67 |
+
{"eval_index": 66, "rl_env_steps": 1326658, "target_step": 1320000, "sr": 0.275, "round": 166, "update_index": 498000, "policy_version": 166, "chunk_transitions": 166000, "gradient_updates": 498000, "n_success": 55, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23699999999999996, "gain": -0.01350000000000004, "observed": true, "imputed": false}
|
| 68 |
+
{"eval_index": 67, "rl_env_steps": 1342626, "target_step": 1340000, "sr": 0.23, "round": 168, "update_index": 504000, "policy_version": 168, "chunk_transitions": 168000, "gradient_updates": 504000, "n_success": 46, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2355, "gain": -0.021000000000000074, "observed": true, "imputed": false}
|
| 69 |
+
{"eval_index": 68, "rl_env_steps": 1366615, "target_step": 1360000, "sr": 0.27, "round": 171, "update_index": 513000, "policy_version": 171, "chunk_transitions": 171000, "gradient_updates": 513000, "n_success": 54, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.238, "gain": -0.017000000000000015, "observed": true, "imputed": false}
|
| 70 |
+
{"eval_index": 69, "rl_env_steps": 1382606, "target_step": 1380000, "sr": 0.175, "round": 173, "update_index": 519000, "policy_version": 173, "chunk_transitions": 173000, "gradient_updates": 519000, "n_success": 35, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23249999999999998, "gain": -0.018500000000000016, "observed": true, "imputed": false}
|
| 71 |
+
{"eval_index": 70, "rl_env_steps": 1406584, "target_step": 1400000, "sr": 0.255, "round": 176, "update_index": 528000, "policy_version": 176, "chunk_transitions": 176000, "gradient_updates": 528000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.236, "gain": -0.012500000000000039, "observed": true, "imputed": false}
|
| 72 |
+
{"eval_index": 71, "rl_env_steps": 1422552, "target_step": 1420000, "sr": 0.155, "round": 178, "update_index": 534000, "policy_version": 178, "chunk_transitions": 178000, "gradient_updates": 534000, "n_success": 31, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23099999999999996, "gain": -0.012500000000000039, "observed": true, "imputed": false}
|
| 73 |
+
{"eval_index": 72, "rl_env_steps": 1446534, "target_step": 1440000, "sr": 0.225, "round": 181, "update_index": 543000, "policy_version": 181, "chunk_transitions": 181000, "gradient_updates": 543000, "n_success": 45, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22749999999999998, "gain": -0.015000000000000013, "observed": true, "imputed": false}
|
| 74 |
+
{"eval_index": 73, "rl_env_steps": 1462512, "target_step": 1460000, "sr": 0.255, "round": 183, "update_index": 549000, "policy_version": 183, "chunk_transitions": 183000, "gradient_updates": 549000, "n_success": 51, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.229, "gain": -0.00949999999999998, "observed": true, "imputed": false}
|
| 75 |
+
{"eval_index": 74, "rl_env_steps": 1486479, "target_step": 1480000, "sr": 0.22, "round": 186, "update_index": 558000, "policy_version": 186, "chunk_transitions": 186000, "gradient_updates": 558000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2305, "gain": -0.004500000000000004, "observed": true, "imputed": false}
|
| 76 |
+
{"eval_index": 75, "rl_env_steps": 1502465, "target_step": 1500000, "sr": 0.195, "round": 188, "update_index": 564000, "policy_version": 188, "chunk_transitions": 188000, "gradient_updates": 564000, "n_success": 39, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22549999999999998, "gain": -0.005000000000000032, "observed": true, "imputed": false}
|
| 77 |
+
{"eval_index": 76, "rl_env_steps": 1526448, "target_step": 1520000, "sr": 0.3, "round": 191, "update_index": 573000, "policy_version": 191, "chunk_transitions": 191000, "gradient_updates": 573000, "n_success": 60, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22799999999999998, "gain": -0.00899999999999998, "observed": true, "imputed": false}
|
| 78 |
+
{"eval_index": 77, "rl_env_steps": 1542429, "target_step": 1540000, "sr": 0.235, "round": 193, "update_index": 579000, "policy_version": 193, "chunk_transitions": 193000, "gradient_updates": 579000, "n_success": 47, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.22849999999999998, "gain": -0.007000000000000006, "observed": true, "imputed": false}
|
| 79 |
+
{"eval_index": 78, "rl_env_steps": 1566419, "target_step": 1560000, "sr": 0.25, "round": 196, "update_index": 588000, "policy_version": 196, "chunk_transitions": 196000, "gradient_updates": 588000, "n_success": 50, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.2265, "gain": -0.011499999999999982, "observed": true, "imputed": false}
|
| 80 |
+
{"eval_index": 79, "rl_env_steps": 1582406, "target_step": 1580000, "sr": 0.245, "round": 198, "update_index": 594000, "policy_version": 198, "chunk_transitions": 198000, "gradient_updates": 594000, "n_success": 49, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23349999999999999, "gain": 0.0010000000000000009, "observed": true, "imputed": false}
|
| 81 |
+
{"eval_index": 80, "rl_env_steps": 1606374, "target_step": 1600000, "sr": 0.22, "round": 201, "update_index": 603000, "policy_version": 201, "chunk_transitions": 201000, "gradient_updates": 603000, "n_success": 44, "episodes": 200, "seed_start": 960000, "success_requested": 0, "success_skipped": 0, "success_rounds_with_skips": 0, "success_skipped_td": 0, "success_skipped_imitation": 0, "success_requested_imitation": 0, "ma": 0.23000000000000004, "gain": -0.00599999999999995, "observed": true, "imputed": false}
|
RL/ToolHang100/QC/diverse100/seed_1/complete/README.md
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Complete run archive
|
| 2 |
+
|
| 3 |
+
See publication.json for identity and terminal state. Download all files in this complete/ directory, then run `python restore.py --archive-dir . --output /path/to/restored`. It checks part hashes and every restored file against manifest.json. zstd command is required for .zst archives.
|
| 4 |
+
|
| 5 |
+
The archive contains run/ (all retained checkpoints, replay, offline buffer, metrics, evaluations, diagnostics and W&B records) and dependencies/ (base BC, manifests, exact observed source and runtime inventory). Original absolute paths remain in historical metadata. Use original directory locations or an explicit recorded path migration before training. Raw demo/image datasets are inputs referenced by their recorded hashes; they are not duplicated in this run-artifact archive. Broken external W&B service-log links are enumerated, never represented as archived bytes.
|
RL/ToolHang100/QC/diverse100/seed_1/complete/SHA256SUMS
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
4407d794e33cd9f5f66d6c6677319a2c4f1059c40dd91cb0b7c51127d75916c0 best/source_metadata.json
|
| 2 |
+
63cdfe34340ccfcfd78db40f4d4defa977b01c010860db6a547d065d7ddf3379 best/source_resolved_config.json
|
| 3 |
+
fc32c5a044147dca30b67057fc8d732e1a35ad69786d41a690ce1977cff0bb57 best/state.pkl
|
| 4 |
+
5073d4ee08c2cfe61160c7fe79b908ffbfbae429aa37896072ddf3eb8a061a07 comparison_evaluations.jsonl
|
| 5 |
+
39c57050e748e4d3f5fd0314a47c05e329d07d4468132cce324735f7fef59b56 complete/README.md
|
| 6 |
+
c38e50e30f0862366b5bc4ceb1d052648b79801784be322560a7d9cdb785bc6d complete/archive.json
|
| 7 |
+
80751fc8d39ede20b7ac35db6eb6d475223b30043cb65cfdcda93e6161fcabf8 complete/full_run.tar.zst.part000
|
| 8 |
+
a4cdb1d2b617dce32ca3b2edb2f0ba6ad0e679d6a25d3831c098d6472fc3ae34 complete/full_run.tar.zst.part001
|
| 9 |
+
e22acd852a5986980083b60e61f8f497a97f395a693530e085b38fd124b9142d complete/full_run.tar.zst.part002
|
| 10 |
+
6636de28dc5fe74734e4f1012c7d484e06a32c661843f476a43b1277a96b0188 complete/full_run.tar.zst.part003
|
| 11 |
+
eea84e68704007648bcdf2ebd0853f89f8dbe1d8bbe7a6dafcb9f5ca4e7e30aa complete/full_run.tar.zst.part004
|
| 12 |
+
4dd765894b61741110210c0a917672db7b1837b74c51bfa3121b1907eac20a18 complete/local_archive_verification.json
|
| 13 |
+
28f1c22eed6777f62ce2b90b5c2b8f62515a354bb1d33491aa3c361dcb673bcf complete/manifest.json
|
| 14 |
+
04bb1f06933b925c0b04bd2d884a973b9498bfa5c900e3f421ab43786fe1f2f1 complete/publication.json
|
| 15 |
+
149a44cd0b002e83f507d1e46d813d5e2b8cf6f9e1bb51f14708343f48788ac2 complete/restore.py
|
| 16 |
+
49bcebbf991b15bc19fa2a7a282bbd4a398ea981588aae431290ddafba936456 convergence.json
|
| 17 |
+
15d3ab051c68c3add53e4a2df366a15f123fefcf81fb879577b247963c2d9614 convergence.jsonl
|
| 18 |
+
1bd80e3bf7d314df54e9e1acf47902ada35684961b3ef9f28271851aec8e3067 eval/final.parquet
|
| 19 |
+
6fb9cf62811b4b258dc9bac5640372d6cde0949eafc41d1216321888f5499da0 final/source_metadata.json
|
| 20 |
+
63cdfe34340ccfcfd78db40f4d4defa977b01c010860db6a547d065d7ddf3379 final/source_resolved_config.json
|
| 21 |
+
8f507af6fb47652d3b5b512d06c8bc51d2da9d2bab494670f88f9b9ce859104d final/state.pkl
|
| 22 |
+
b49df1f510fc5f3a256ed92e202b4b4586349d2888bcfd870c32b573eed8ebb3 frozen_base.json
|
| 23 |
+
3213549f044d53303cb3e14a2a29fa9bc4d758cbfa625dda9f5042120245b169 offline_buffer.json
|
| 24 |
+
63cdfe34340ccfcfd78db40f4d4defa977b01c010860db6a547d065d7ddf3379 resolved_config.json
|
| 25 |
+
5902c5b225975dc3730d6ee7e7daea1bb7740771fc45adcf86bb84e8b47ffc98 start_protocol.json
|
| 26 |
+
cd8c871bfbcfdbadf557f7788171bfcc7c69e130201eff1a4e1544c296400f5c status.json
|
| 27 |
+
4efd9d2898cb2c5ff54f2c067a965943274cee8b16755d1b44d9ca78794e9973 training_accounting.json
|
RL/ToolHang100/QC/diverse100/seed_1/complete/archive.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"format": "tar.zst",
|
| 3 |
+
"parts": [
|
| 4 |
+
{
|
| 5 |
+
"path": "full_run.tar.zst.part000",
|
| 6 |
+
"size": 1073741824,
|
| 7 |
+
"sha256": "80751fc8d39ede20b7ac35db6eb6d475223b30043cb65cfdcda93e6161fcabf8"
|
| 8 |
+
},
|
| 9 |
+
{
|
| 10 |
+
"path": "full_run.tar.zst.part001",
|
| 11 |
+
"size": 1073741824,
|
| 12 |
+
"sha256": "a4cdb1d2b617dce32ca3b2edb2f0ba6ad0e679d6a25d3831c098d6472fc3ae34"
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"path": "full_run.tar.zst.part002",
|
| 16 |
+
"size": 1073741824,
|
| 17 |
+
"sha256": "e22acd852a5986980083b60e61f8f497a97f395a693530e085b38fd124b9142d"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"path": "full_run.tar.zst.part003",
|
| 21 |
+
"size": 1073741824,
|
| 22 |
+
"sha256": "6636de28dc5fe74734e4f1012c7d484e06a32c661843f476a43b1277a96b0188"
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"path": "full_run.tar.zst.part004",
|
| 26 |
+
"size": 107952165,
|
| 27 |
+
"sha256": "eea84e68704007648bcdf2ebd0853f89f8dbe1d8bbe7a6dafcb9f5ca4e7e30aa"
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"manifest_sha256": "28f1c22eed6777f62ce2b90b5c2b8f62515a354bb1d33491aa3c361dcb673bcf",
|
| 31 |
+
"hardlink_content_dedup": true,
|
| 32 |
+
"member_count": 5883
|
| 33 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part000
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:80751fc8d39ede20b7ac35db6eb6d475223b30043cb65cfdcda93e6161fcabf8
|
| 3 |
+
size 1073741824
|
RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part001
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a4cdb1d2b617dce32ca3b2edb2f0ba6ad0e679d6a25d3831c098d6472fc3ae34
|
| 3 |
+
size 1073741824
|
RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part002
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e22acd852a5986980083b60e61f8f497a97f395a693530e085b38fd124b9142d
|
| 3 |
+
size 1073741824
|
RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part003
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6636de28dc5fe74734e4f1012c7d484e06a32c661843f476a43b1277a96b0188
|
| 3 |
+
size 1073741824
|
RL/ToolHang100/QC/diverse100/seed_1/complete/full_run.tar.zst.part004
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eea84e68704007648bcdf2ebd0853f89f8dbe1d8bbe7a6dafcb9f5ca4e7e30aa
|
| 3 |
+
size 107952165
|
RL/ToolHang100/QC/diverse100/seed_1/complete/local_archive_verification.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"passed": true,
|
| 3 |
+
"verified_members": 5882,
|
| 4 |
+
"verified_archive_parts": 5,
|
| 5 |
+
"time": 1793313462.6377976
|
| 6 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/complete/manifest.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
RL/ToolHang100/QC/diverse100/seed_1/complete/publication.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"run_id": "23871f6e",
|
| 4 |
+
"task": "ToolHang100",
|
| 5 |
+
"algorithm": "QC",
|
| 6 |
+
"variant": "qc",
|
| 7 |
+
"cohort": "diverse100",
|
| 8 |
+
"seed": 1,
|
| 9 |
+
"protocol": "study2_protocol_v10",
|
| 10 |
+
"status": "budget_complete",
|
| 11 |
+
"path": "RL/ToolHang100/QC/diverse100/seed_1",
|
| 12 |
+
"checkpoints": {
|
| 13 |
+
"best": {
|
| 14 |
+
"path": "RL/ToolHang100/QC/diverse100/seed_1/best/state.pkl",
|
| 15 |
+
"sha256": "fc32c5a044147dca30b67057fc8d732e1a35ad69786d41a690ce1977cff0bb57",
|
| 16 |
+
"bytes": 207200290,
|
| 17 |
+
"source_path": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261024_toolhang_diverse100_qc_v10_seed1/checkpoints/best/eval_00004_step_87901_session_1792820716501319935/checkpoint/state.pkl",
|
| 18 |
+
"evaluation": {
|
| 19 |
+
"chunk_transitions": 11000,
|
| 20 |
+
"episodes": 200,
|
| 21 |
+
"eval_index": 4,
|
| 22 |
+
"gain": null,
|
| 23 |
+
"gradient_updates": 33000,
|
| 24 |
+
"ma": null,
|
| 25 |
+
"n_success": 67,
|
| 26 |
+
"policy_version": 11,
|
| 27 |
+
"rl_env_steps": 87901,
|
| 28 |
+
"round": 11,
|
| 29 |
+
"seed_start": 960000,
|
| 30 |
+
"sr": 0.335,
|
| 31 |
+
"success_requested": 0,
|
| 32 |
+
"success_requested_imitation": 0,
|
| 33 |
+
"success_rounds_with_skips": 0,
|
| 34 |
+
"success_skipped": 0,
|
| 35 |
+
"success_skipped_imitation": 0,
|
| 36 |
+
"success_skipped_td": 0,
|
| 37 |
+
"target_step": 80000,
|
| 38 |
+
"update_index": 33000
|
| 39 |
+
}
|
| 40 |
+
},
|
| 41 |
+
"final": {
|
| 42 |
+
"path": "RL/ToolHang100/QC/diverse100/seed_1/final/state.pkl",
|
| 43 |
+
"sha256": "8f507af6fb47652d3b5b512d06c8bc51d2da9d2bab494670f88f9b9ce859104d",
|
| 44 |
+
"bytes": 207222745,
|
| 45 |
+
"source_path": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261024_toolhang_diverse100_qc_v10_seed1/checkpoints/final/session_1792820716501319935/checkpoint/state.pkl",
|
| 46 |
+
"evaluation": {
|
| 47 |
+
"episodes": 300,
|
| 48 |
+
"imputed": false,
|
| 49 |
+
"n_success": 77,
|
| 50 |
+
"observed": true,
|
| 51 |
+
"seed_start": 800000,
|
| 52 |
+
"sr": 0.25666666666666665
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"complete_artifacts": "RL/ToolHang100/QC/diverse100/seed_1/complete",
|
| 57 |
+
"source_host": "Paratera",
|
| 58 |
+
"source_run": "/ssd/WorkSpace/RL-Postrain/runs/experiments/study2/pixel_state/20261024_toolhang_diverse100_qc_v10_seed1",
|
| 59 |
+
"artifact_files": 5882,
|
| 60 |
+
"archive_bytes": 4402919461,
|
| 61 |
+
"omitted_broken_log_links": 0
|
| 62 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/complete/restore.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Restore a full RL run and verify every archived member without executing its contents."""
|
| 2 |
+
import argparse,hashlib,json,os,subprocess,tarfile,threading,shutil
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
def digest(p):
|
| 6 |
+
h=hashlib.sha256()
|
| 7 |
+
with p.open('rb') as f:
|
| 8 |
+
for b in iter(lambda:f.read(8*1024**2),b''):h.update(b)
|
| 9 |
+
return h.hexdigest()
|
| 10 |
+
def main():
|
| 11 |
+
ap=argparse.ArgumentParser();ap.add_argument('--archive-dir',type=Path,default=Path('.'));ap.add_argument('--output',type=Path,required=True);args=ap.parse_args();base=args.archive_dir;out=args.output.resolve();out.mkdir(parents=True,exist_ok=True);assert not any(out.iterdir()),'Use an empty output directory'
|
| 12 |
+
arc=json.loads((base/'archive.json').read_text());assert digest(base/'manifest.json')==arc['manifest_sha256'];manifest=json.loads((base/'manifest.json').read_text());expected={e['path']:e for e in manifest['entries']}
|
| 13 |
+
for part in arc['parts']:
|
| 14 |
+
p=base/part['path'];assert p.stat().st_size==part['size'] and digest(p)==part['sha256'],p
|
| 15 |
+
cmd=['zstd','-q','-d','-c'] if arc['format'].endswith('zst') else ['gzip','-d','-c'];proc=subprocess.Popen(cmd,stdin=subprocess.PIPE,stdout=subprocess.PIPE);errors=[]
|
| 16 |
+
def feed():
|
| 17 |
+
try:
|
| 18 |
+
for part in arc['parts']:
|
| 19 |
+
with (base/part['path']).open('rb') as f:shutil.copyfileobj(f,proc.stdin,1024**2)
|
| 20 |
+
except BaseException as e:errors.append(e)
|
| 21 |
+
finally:proc.stdin.close()
|
| 22 |
+
t=threading.Thread(target=feed);t.start();seen={}
|
| 23 |
+
with tarfile.open(fileobj=proc.stdout,mode='r|') as tar:
|
| 24 |
+
for m in tar:
|
| 25 |
+
if m.name=='MANIFEST.json':assert hashlib.sha256(tar.extractfile(m).read()).hexdigest()==arc['manifest_sha256'];continue
|
| 26 |
+
e=expected[m.name];dest=out/m.name;assert dest.resolve().is_relative_to(out);dest.parent.mkdir(parents=True,exist_ok=True)
|
| 27 |
+
if m.issym():
|
| 28 |
+
assert e['type']=='symlink' and m.linkname==e['target'] and (dest.parent/m.linkname).resolve().is_relative_to(out);dest.symlink_to(m.linkname);seen[m.name]=None
|
| 29 |
+
elif m.islnk():
|
| 30 |
+
assert seen[m.linkname]==e['sha256'];src=out/m.linkname;assert src.resolve().is_relative_to(out);os.link(src,dest);seen[m.name]=e['sha256']
|
| 31 |
+
else:
|
| 32 |
+
assert m.isfile() and m.size==e['size'];h=hashlib.sha256()
|
| 33 |
+
with dest.open('xb') as f:
|
| 34 |
+
inp=tar.extractfile(m)
|
| 35 |
+
for b in iter(lambda:inp.read(8*1024**2),b''):f.write(b);h.update(b)
|
| 36 |
+
assert h.hexdigest()==e['sha256'],m.name;dest.chmod(e.get('mode',0o644));seen[m.name]=h.hexdigest()
|
| 37 |
+
t.join();assert proc.wait()==0 and not errors and len(seen)==len(expected);shutil.copy2(base/'manifest.json',out/'MANIFEST.json');(out/'RESTORE_VERIFIED.json').write_text(json.dumps({'passed':True,'files':len(seen),'run_id':manifest['run']['run_id']},indent=2));print('RESTORE VERIFIED',manifest['run']['run_id'],len(seen),'members')
|
| 38 |
+
if __name__=='__main__':main()
|
RL/ToolHang100/QC/diverse100/seed_1/convergence.json
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"evals": 81,
|
| 3 |
+
"final_eval": {
|
| 4 |
+
"episodes": 300,
|
| 5 |
+
"imputed": false,
|
| 6 |
+
"n_success": 77,
|
| 7 |
+
"observed": true,
|
| 8 |
+
"seed_start": 800000,
|
| 9 |
+
"sr": 0.25666666666666665,
|
| 10 |
+
"successes": [
|
| 11 |
+
0,
|
| 12 |
+
1,
|
| 13 |
+
0,
|
| 14 |
+
1,
|
| 15 |
+
0,
|
| 16 |
+
0,
|
| 17 |
+
0,
|
| 18 |
+
0,
|
| 19 |
+
1,
|
| 20 |
+
0,
|
| 21 |
+
0,
|
| 22 |
+
0,
|
| 23 |
+
0,
|
| 24 |
+
1,
|
| 25 |
+
1,
|
| 26 |
+
0,
|
| 27 |
+
0,
|
| 28 |
+
1,
|
| 29 |
+
0,
|
| 30 |
+
0,
|
| 31 |
+
0,
|
| 32 |
+
0,
|
| 33 |
+
1,
|
| 34 |
+
0,
|
| 35 |
+
1,
|
| 36 |
+
0,
|
| 37 |
+
0,
|
| 38 |
+
0,
|
| 39 |
+
1,
|
| 40 |
+
0,
|
| 41 |
+
0,
|
| 42 |
+
0,
|
| 43 |
+
0,
|
| 44 |
+
0,
|
| 45 |
+
0,
|
| 46 |
+
0,
|
| 47 |
+
0,
|
| 48 |
+
0,
|
| 49 |
+
0,
|
| 50 |
+
0,
|
| 51 |
+
1,
|
| 52 |
+
0,
|
| 53 |
+
0,
|
| 54 |
+
1,
|
| 55 |
+
0,
|
| 56 |
+
0,
|
| 57 |
+
0,
|
| 58 |
+
0,
|
| 59 |
+
0,
|
| 60 |
+
0,
|
| 61 |
+
0,
|
| 62 |
+
0,
|
| 63 |
+
0,
|
| 64 |
+
0,
|
| 65 |
+
0,
|
| 66 |
+
0,
|
| 67 |
+
0,
|
| 68 |
+
0,
|
| 69 |
+
0,
|
| 70 |
+
0,
|
| 71 |
+
0,
|
| 72 |
+
0,
|
| 73 |
+
0,
|
| 74 |
+
1,
|
| 75 |
+
1,
|
| 76 |
+
1,
|
| 77 |
+
0,
|
| 78 |
+
0,
|
| 79 |
+
0,
|
| 80 |
+
1,
|
| 81 |
+
0,
|
| 82 |
+
1,
|
| 83 |
+
0,
|
| 84 |
+
0,
|
| 85 |
+
0,
|
| 86 |
+
0,
|
| 87 |
+
0,
|
| 88 |
+
0,
|
| 89 |
+
0,
|
| 90 |
+
1,
|
| 91 |
+
0,
|
| 92 |
+
0,
|
| 93 |
+
0,
|
| 94 |
+
0,
|
| 95 |
+
0,
|
| 96 |
+
0,
|
| 97 |
+
0,
|
| 98 |
+
0,
|
| 99 |
+
0,
|
| 100 |
+
0,
|
| 101 |
+
0,
|
| 102 |
+
0,
|
| 103 |
+
0,
|
| 104 |
+
1,
|
| 105 |
+
1,
|
| 106 |
+
0,
|
| 107 |
+
0,
|
| 108 |
+
0,
|
| 109 |
+
0,
|
| 110 |
+
0,
|
| 111 |
+
0,
|
| 112 |
+
0,
|
| 113 |
+
1,
|
| 114 |
+
0,
|
| 115 |
+
1,
|
| 116 |
+
1,
|
| 117 |
+
0,
|
| 118 |
+
0,
|
| 119 |
+
1,
|
| 120 |
+
0,
|
| 121 |
+
0,
|
| 122 |
+
0,
|
| 123 |
+
0,
|
| 124 |
+
0,
|
| 125 |
+
0,
|
| 126 |
+
0,
|
| 127 |
+
1,
|
| 128 |
+
0,
|
| 129 |
+
0,
|
| 130 |
+
0,
|
| 131 |
+
0,
|
| 132 |
+
0,
|
| 133 |
+
1,
|
| 134 |
+
0,
|
| 135 |
+
0,
|
| 136 |
+
0,
|
| 137 |
+
0,
|
| 138 |
+
0,
|
| 139 |
+
0,
|
| 140 |
+
0,
|
| 141 |
+
0,
|
| 142 |
+
0,
|
| 143 |
+
0,
|
| 144 |
+
0,
|
| 145 |
+
1,
|
| 146 |
+
0,
|
| 147 |
+
0,
|
| 148 |
+
0,
|
| 149 |
+
0,
|
| 150 |
+
0,
|
| 151 |
+
1,
|
| 152 |
+
0,
|
| 153 |
+
0,
|
| 154 |
+
0,
|
| 155 |
+
0,
|
| 156 |
+
0,
|
| 157 |
+
0,
|
| 158 |
+
0,
|
| 159 |
+
1,
|
| 160 |
+
0,
|
| 161 |
+
0,
|
| 162 |
+
1,
|
| 163 |
+
0,
|
| 164 |
+
0,
|
| 165 |
+
1,
|
| 166 |
+
0,
|
| 167 |
+
0,
|
| 168 |
+
0,
|
| 169 |
+
0,
|
| 170 |
+
0,
|
| 171 |
+
1,
|
| 172 |
+
1,
|
| 173 |
+
0,
|
| 174 |
+
1,
|
| 175 |
+
0,
|
| 176 |
+
0,
|
| 177 |
+
0,
|
| 178 |
+
0,
|
| 179 |
+
1,
|
| 180 |
+
0,
|
| 181 |
+
1,
|
| 182 |
+
1,
|
| 183 |
+
0,
|
| 184 |
+
0,
|
| 185 |
+
0,
|
| 186 |
+
0,
|
| 187 |
+
0,
|
| 188 |
+
0,
|
| 189 |
+
0,
|
| 190 |
+
1,
|
| 191 |
+
0,
|
| 192 |
+
1,
|
| 193 |
+
0,
|
| 194 |
+
0,
|
| 195 |
+
0,
|
| 196 |
+
1,
|
| 197 |
+
1,
|
| 198 |
+
0,
|
| 199 |
+
1,
|
| 200 |
+
0,
|
| 201 |
+
0,
|
| 202 |
+
0,
|
| 203 |
+
0,
|
| 204 |
+
0,
|
| 205 |
+
0,
|
| 206 |
+
1,
|
| 207 |
+
1,
|
| 208 |
+
1,
|
| 209 |
+
0,
|
| 210 |
+
0,
|
| 211 |
+
1,
|
| 212 |
+
0,
|
| 213 |
+
0,
|
| 214 |
+
0,
|
| 215 |
+
0,
|
| 216 |
+
1,
|
| 217 |
+
1,
|
| 218 |
+
0,
|
| 219 |
+
0,
|
| 220 |
+
0,
|
| 221 |
+
0,
|
| 222 |
+
1,
|
| 223 |
+
1,
|
| 224 |
+
0,
|
| 225 |
+
0,
|
| 226 |
+
1,
|
| 227 |
+
1,
|
| 228 |
+
0,
|
| 229 |
+
0,
|
| 230 |
+
0,
|
| 231 |
+
0,
|
| 232 |
+
0,
|
| 233 |
+
0,
|
| 234 |
+
0,
|
| 235 |
+
0,
|
| 236 |
+
1,
|
| 237 |
+
0,
|
| 238 |
+
0,
|
| 239 |
+
1,
|
| 240 |
+
0,
|
| 241 |
+
1,
|
| 242 |
+
1,
|
| 243 |
+
0,
|
| 244 |
+
0,
|
| 245 |
+
0,
|
| 246 |
+
1,
|
| 247 |
+
0,
|
| 248 |
+
0,
|
| 249 |
+
0,
|
| 250 |
+
0,
|
| 251 |
+
0,
|
| 252 |
+
0,
|
| 253 |
+
1,
|
| 254 |
+
1,
|
| 255 |
+
1,
|
| 256 |
+
0,
|
| 257 |
+
0,
|
| 258 |
+
0,
|
| 259 |
+
0,
|
| 260 |
+
0,
|
| 261 |
+
0,
|
| 262 |
+
0,
|
| 263 |
+
0,
|
| 264 |
+
1,
|
| 265 |
+
0,
|
| 266 |
+
1,
|
| 267 |
+
1,
|
| 268 |
+
0,
|
| 269 |
+
0,
|
| 270 |
+
1,
|
| 271 |
+
0,
|
| 272 |
+
1,
|
| 273 |
+
1,
|
| 274 |
+
0,
|
| 275 |
+
0,
|
| 276 |
+
0,
|
| 277 |
+
1,
|
| 278 |
+
0,
|
| 279 |
+
0,
|
| 280 |
+
0,
|
| 281 |
+
0,
|
| 282 |
+
1,
|
| 283 |
+
0,
|
| 284 |
+
0,
|
| 285 |
+
1,
|
| 286 |
+
1,
|
| 287 |
+
0,
|
| 288 |
+
0,
|
| 289 |
+
0,
|
| 290 |
+
0,
|
| 291 |
+
0,
|
| 292 |
+
0,
|
| 293 |
+
0,
|
| 294 |
+
1,
|
| 295 |
+
1,
|
| 296 |
+
0,
|
| 297 |
+
1,
|
| 298 |
+
0,
|
| 299 |
+
1,
|
| 300 |
+
0,
|
| 301 |
+
0,
|
| 302 |
+
1,
|
| 303 |
+
1,
|
| 304 |
+
0,
|
| 305 |
+
0,
|
| 306 |
+
0,
|
| 307 |
+
1,
|
| 308 |
+
1,
|
| 309 |
+
0,
|
| 310 |
+
0
|
| 311 |
+
]
|
| 312 |
+
},
|
| 313 |
+
"last": {
|
| 314 |
+
"chunk_transitions": 201000,
|
| 315 |
+
"episodes": 200,
|
| 316 |
+
"eval_index": 80,
|
| 317 |
+
"gain": -0.00599999999999995,
|
| 318 |
+
"gradient_updates": 603000,
|
| 319 |
+
"ma": 0.23000000000000004,
|
| 320 |
+
"n_success": 44,
|
| 321 |
+
"policy_version": 201,
|
| 322 |
+
"rl_env_steps": 1606374,
|
| 323 |
+
"round": 201,
|
| 324 |
+
"seed_start": 960000,
|
| 325 |
+
"sr": 0.22,
|
| 326 |
+
"success_requested": 0,
|
| 327 |
+
"success_requested_imitation": 0,
|
| 328 |
+
"success_rounds_with_skips": 0,
|
| 329 |
+
"success_skipped": 0,
|
| 330 |
+
"success_skipped_imitation": 0,
|
| 331 |
+
"success_skipped_td": 0,
|
| 332 |
+
"target_step": 1600000,
|
| 333 |
+
"update_index": 603000
|
| 334 |
+
},
|
| 335 |
+
"rule": {
|
| 336 |
+
"early_stop": false,
|
| 337 |
+
"every_env_steps": 20000,
|
| 338 |
+
"gain_lag": 10,
|
| 339 |
+
"high_sr": 0.95,
|
| 340 |
+
"high_sr_evals": 5,
|
| 341 |
+
"include_step0": true,
|
| 342 |
+
"ma_window": 10,
|
| 343 |
+
"max_env_steps": 1606196,
|
| 344 |
+
"min_env_steps": 200000,
|
| 345 |
+
"min_gain": 0.02,
|
| 346 |
+
"zero_success_evals": 8
|
| 347 |
+
},
|
| 348 |
+
"status": "budget_complete",
|
| 349 |
+
"stop": {
|
| 350 |
+
"eval_index": 80,
|
| 351 |
+
"gain": -0.00599999999999995,
|
| 352 |
+
"ma": 0.23000000000000004,
|
| 353 |
+
"rl_env_steps": 1606374,
|
| 354 |
+
"sr": 0.22,
|
| 355 |
+
"status": "budget_complete",
|
| 356 |
+
"trend": "flat"
|
| 357 |
+
},
|
| 358 |
+
"would_stop": {
|
| 359 |
+
"eval_index": 19,
|
| 360 |
+
"gain": -0.017499999999999988,
|
| 361 |
+
"ma": 0.23150000000000004,
|
| 362 |
+
"rl_env_steps": 383610,
|
| 363 |
+
"sr": 0.24,
|
| 364 |
+
"status": "converged_plateau",
|
| 365 |
+
"trend": "flat"
|
| 366 |
+
}
|
| 367 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/convergence.jsonl
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"chunk_transitions": 0, "episodes": 200, "eval_index": 0, "gain": null, "gradient_updates": 0, "ma": null, "n_success": 38, "policy_version": 0, "rl_env_steps": 0, "round": 0, "seed_start": 960000, "sr": 0.19, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 0, "update_index": 0}
|
| 2 |
+
{"chunk_transitions": 3000, "episodes": 200, "eval_index": 1, "gain": null, "gradient_updates": 9000, "ma": null, "n_success": 49, "policy_version": 3, "rl_env_steps": 23971, "round": 3, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 20000, "update_index": 9000}
|
| 3 |
+
{"chunk_transitions": 6000, "episodes": 200, "eval_index": 2, "gain": null, "gradient_updates": 18000, "ma": null, "n_success": 44, "policy_version": 6, "rl_env_steps": 47944, "round": 6, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 40000, "update_index": 18000}
|
| 4 |
+
{"chunk_transitions": 8000, "episodes": 200, "eval_index": 3, "gain": null, "gradient_updates": 24000, "ma": null, "n_success": 54, "policy_version": 8, "rl_env_steps": 63925, "round": 8, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 60000, "update_index": 24000}
|
| 5 |
+
{"chunk_transitions": 11000, "episodes": 200, "eval_index": 4, "gain": null, "gradient_updates": 33000, "ma": null, "n_success": 67, "policy_version": 11, "rl_env_steps": 87901, "round": 11, "seed_start": 960000, "sr": 0.335, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 80000, "update_index": 33000}
|
| 6 |
+
{"chunk_transitions": 13000, "episodes": 200, "eval_index": 5, "gain": null, "gradient_updates": 39000, "ma": null, "n_success": 52, "policy_version": 13, "rl_env_steps": 103895, "round": 13, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 100000, "update_index": 39000}
|
| 7 |
+
{"chunk_transitions": 16000, "episodes": 200, "eval_index": 6, "gain": null, "gradient_updates": 48000, "ma": null, "n_success": 32, "policy_version": 16, "rl_env_steps": 127876, "round": 16, "seed_start": 960000, "sr": 0.16, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 120000, "update_index": 48000}
|
| 8 |
+
{"chunk_transitions": 18000, "episodes": 200, "eval_index": 7, "gain": null, "gradient_updates": 54000, "ma": null, "n_success": 50, "policy_version": 18, "rl_env_steps": 143854, "round": 18, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 140000, "update_index": 54000}
|
| 9 |
+
{"chunk_transitions": 21000, "episodes": 200, "eval_index": 8, "gain": null, "gradient_updates": 63000, "ma": null, "n_success": 59, "policy_version": 21, "rl_env_steps": 167826, "round": 21, "seed_start": 960000, "sr": 0.295, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 160000, "update_index": 63000}
|
| 10 |
+
{"chunk_transitions": 23000, "episodes": 200, "eval_index": 9, "gain": null, "gradient_updates": 69000, "ma": 0.24900000000000003, "n_success": 53, "policy_version": 23, "rl_env_steps": 183814, "round": 23, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 180000, "update_index": 69000}
|
| 11 |
+
{"chunk_transitions": 26000, "episodes": 200, "eval_index": 10, "gain": null, "gradient_updates": 78000, "ma": 0.25149999999999995, "n_success": 43, "policy_version": 26, "rl_env_steps": 207787, "round": 26, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 200000, "update_index": 78000}
|
| 12 |
+
{"chunk_transitions": 28000, "episodes": 200, "eval_index": 11, "gain": null, "gradient_updates": 84000, "ma": 0.25, "n_success": 46, "policy_version": 28, "rl_env_steps": 223769, "round": 28, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 220000, "update_index": 84000}
|
| 13 |
+
{"chunk_transitions": 31000, "episodes": 200, "eval_index": 12, "gain": null, "gradient_updates": 93000, "ma": 0.24749999999999997, "n_success": 39, "policy_version": 31, "rl_env_steps": 247753, "round": 31, "seed_start": 960000, "sr": 0.195, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 240000, "update_index": 93000}
|
| 14 |
+
{"chunk_transitions": 33000, "episodes": 200, "eval_index": 13, "gain": null, "gradient_updates": 99000, "ma": 0.24549999999999997, "n_success": 50, "policy_version": 33, "rl_env_steps": 263739, "round": 33, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 260000, "update_index": 99000}
|
| 15 |
+
{"chunk_transitions": 36000, "episodes": 200, "eval_index": 14, "gain": null, "gradient_updates": 108000, "ma": 0.23700000000000002, "n_success": 50, "policy_version": 36, "rl_env_steps": 287729, "round": 36, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 280000, "update_index": 108000}
|
| 16 |
+
{"chunk_transitions": 38000, "episodes": 200, "eval_index": 15, "gain": null, "gradient_updates": 114000, "ma": 0.23150000000000004, "n_success": 41, "policy_version": 38, "rl_env_steps": 303714, "round": 38, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 300000, "update_index": 114000}
|
| 17 |
+
{"chunk_transitions": 41000, "episodes": 200, "eval_index": 16, "gain": null, "gradient_updates": 123000, "ma": 0.2425, "n_success": 54, "policy_version": 41, "rl_env_steps": 327675, "round": 41, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 320000, "update_index": 123000}
|
| 18 |
+
{"chunk_transitions": 43000, "episodes": 200, "eval_index": 17, "gain": null, "gradient_updates": 129000, "ma": 0.2375, "n_success": 40, "policy_version": 43, "rl_env_steps": 343660, "round": 43, "seed_start": 960000, "sr": 0.2, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 340000, "update_index": 129000}
|
| 19 |
+
{"chunk_transitions": 46000, "episodes": 200, "eval_index": 18, "gain": null, "gradient_updates": 138000, "ma": 0.23399999999999999, "n_success": 52, "policy_version": 46, "rl_env_steps": 367622, "round": 46, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 360000, "update_index": 138000}
|
| 20 |
+
{"chunk_transitions": 48000, "episodes": 200, "eval_index": 19, "gain": -0.017499999999999988, "gradient_updates": 144000, "ma": 0.23150000000000004, "n_success": 48, "policy_version": 48, "rl_env_steps": 383610, "round": 48, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 380000, "update_index": 144000}
|
| 21 |
+
{"chunk_transitions": 51000, "episodes": 200, "eval_index": 20, "gain": -0.020499999999999935, "gradient_updates": 153000, "ma": 0.231, "n_success": 42, "policy_version": 51, "rl_env_steps": 407593, "round": 51, "seed_start": 960000, "sr": 0.21, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 400000, "update_index": 153000}
|
| 22 |
+
{"chunk_transitions": 53000, "episodes": 200, "eval_index": 21, "gain": -0.010000000000000009, "gradient_updates": 159000, "ma": 0.24, "n_success": 64, "policy_version": 53, "rl_env_steps": 423591, "round": 53, "seed_start": 960000, "sr": 0.32, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 420000, "update_index": 159000}
|
| 23 |
+
{"chunk_transitions": 56000, "episodes": 200, "eval_index": 22, "gain": -0.001500000000000029, "gradient_updates": 168000, "ma": 0.24599999999999994, "n_success": 51, "policy_version": 56, "rl_env_steps": 447565, "round": 56, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 440000, "update_index": 168000}
|
| 24 |
+
{"chunk_transitions": 58000, "episodes": 200, "eval_index": 23, "gain": 0.0015000000000000013, "gradient_updates": 174000, "ma": 0.24699999999999997, "n_success": 52, "policy_version": 58, "rl_env_steps": 463565, "round": 58, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 460000, "update_index": 174000}
|
| 25 |
+
{"chunk_transitions": 61000, "episodes": 200, "eval_index": 24, "gain": 0.012999999999999984, "gradient_updates": 183000, "ma": 0.25, "n_success": 56, "policy_version": 61, "rl_env_steps": 487539, "round": 61, "seed_start": 960000, "sr": 0.28, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 480000, "update_index": 183000}
|
| 26 |
+
{"chunk_transitions": 63000, "episodes": 200, "eval_index": 25, "gain": 0.023499999999999965, "gradient_updates": 189000, "ma": 0.255, "n_success": 51, "policy_version": 63, "rl_env_steps": 503527, "round": 63, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 500000, "update_index": 189000}
|
| 27 |
+
{"chunk_transitions": 66000, "episodes": 200, "eval_index": 26, "gain": 0.006000000000000005, "gradient_updates": 198000, "ma": 0.2485, "n_success": 41, "policy_version": 66, "rl_env_steps": 527509, "round": 66, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 520000, "update_index": 198000}
|
| 28 |
+
{"chunk_transitions": 68000, "episodes": 200, "eval_index": 27, "gain": 0.020500000000000018, "gradient_updates": 204000, "ma": 0.258, "n_success": 59, "policy_version": 68, "rl_env_steps": 543493, "round": 68, "seed_start": 960000, "sr": 0.295, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 540000, "update_index": 204000}
|
| 29 |
+
{"chunk_transitions": 71000, "episodes": 200, "eval_index": 28, "gain": 0.02100000000000002, "gradient_updates": 213000, "ma": 0.255, "n_success": 46, "policy_version": 71, "rl_env_steps": 567458, "round": 71, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 560000, "update_index": 213000}
|
| 30 |
+
{"chunk_transitions": 73000, "episodes": 200, "eval_index": 29, "gain": 0.024499999999999966, "gradient_updates": 219000, "ma": 0.256, "n_success": 50, "policy_version": 73, "rl_env_steps": 583442, "round": 73, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 580000, "update_index": 219000}
|
| 31 |
+
{"chunk_transitions": 76000, "episodes": 200, "eval_index": 30, "gain": 0.025499999999999995, "gradient_updates": 228000, "ma": 0.2565, "n_success": 43, "policy_version": 76, "rl_env_steps": 607421, "round": 76, "seed_start": 960000, "sr": 0.215, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 600000, "update_index": 228000}
|
| 32 |
+
{"chunk_transitions": 78000, "episodes": 200, "eval_index": 31, "gain": 0.013500000000000012, "gradient_updates": 234000, "ma": 0.2535, "n_success": 58, "policy_version": 78, "rl_env_steps": 623412, "round": 78, "seed_start": 960000, "sr": 0.29, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 620000, "update_index": 234000}
|
| 33 |
+
{"chunk_transitions": 81000, "episodes": 200, "eval_index": 32, "gain": -0.0014999999999999458, "gradient_updates": 243000, "ma": 0.2445, "n_success": 33, "policy_version": 81, "rl_env_steps": 647389, "round": 81, "seed_start": 960000, "sr": 0.165, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 640000, "update_index": 243000}
|
| 34 |
+
{"chunk_transitions": 83000, "episodes": 200, "eval_index": 33, "gain": -0.0019999999999999463, "gradient_updates": 249000, "ma": 0.24500000000000002, "n_success": 53, "policy_version": 83, "rl_env_steps": 663375, "round": 83, "seed_start": 960000, "sr": 0.265, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 660000, "update_index": 249000}
|
| 35 |
+
{"chunk_transitions": 86000, "episodes": 200, "eval_index": 34, "gain": -0.012999999999999984, "gradient_updates": 258000, "ma": 0.23700000000000002, "n_success": 40, "policy_version": 86, "rl_env_steps": 687326, "round": 86, "seed_start": 960000, "sr": 0.2, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 680000, "update_index": 258000}
|
| 36 |
+
{"chunk_transitions": 88000, "episodes": 200, "eval_index": 35, "gain": -0.01849999999999999, "gradient_updates": 264000, "ma": 0.23650000000000002, "n_success": 50, "policy_version": 88, "rl_env_steps": 703314, "round": 88, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 700000, "update_index": 264000}
|
| 37 |
+
{"chunk_transitions": 91000, "episodes": 200, "eval_index": 36, "gain": -0.007000000000000006, "gradient_updates": 273000, "ma": 0.2415, "n_success": 51, "policy_version": 91, "rl_env_steps": 727280, "round": 91, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 720000, "update_index": 273000}
|
| 38 |
+
{"chunk_transitions": 93000, "episodes": 200, "eval_index": 37, "gain": -0.017499999999999988, "gradient_updates": 279000, "ma": 0.24050000000000002, "n_success": 57, "policy_version": 93, "rl_env_steps": 743258, "round": 93, "seed_start": 960000, "sr": 0.285, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 740000, "update_index": 279000}
|
| 39 |
+
{"chunk_transitions": 96000, "episodes": 200, "eval_index": 38, "gain": -0.018500000000000016, "gradient_updates": 288000, "ma": 0.2365, "n_success": 38, "policy_version": 96, "rl_env_steps": 767232, "round": 96, "seed_start": 960000, "sr": 0.19, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 760000, "update_index": 288000}
|
| 40 |
+
{"chunk_transitions": 98000, "episodes": 200, "eval_index": 39, "gain": -0.017499999999999988, "gradient_updates": 294000, "ma": 0.23850000000000002, "n_success": 54, "policy_version": 98, "rl_env_steps": 783208, "round": 98, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 780000, "update_index": 294000}
|
| 41 |
+
{"chunk_transitions": 101000, "episodes": 200, "eval_index": 40, "gain": -0.015000000000000013, "gradient_updates": 303000, "ma": 0.2415, "n_success": 49, "policy_version": 101, "rl_env_steps": 807164, "round": 101, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 800000, "update_index": 303000}
|
| 42 |
+
{"chunk_transitions": 103000, "episodes": 200, "eval_index": 41, "gain": -0.01899999999999999, "gradient_updates": 309000, "ma": 0.23450000000000001, "n_success": 44, "policy_version": 103, "rl_env_steps": 823128, "round": 103, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 820000, "update_index": 309000}
|
| 43 |
+
{"chunk_transitions": 106000, "episodes": 200, "eval_index": 42, "gain": -0.00649999999999995, "gradient_updates": 318000, "ma": 0.23800000000000004, "n_success": 40, "policy_version": 106, "rl_env_steps": 847114, "round": 106, "seed_start": 960000, "sr": 0.2, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 840000, "update_index": 318000}
|
| 44 |
+
{"chunk_transitions": 108000, "episodes": 200, "eval_index": 43, "gain": -0.004500000000000004, "gradient_updates": 324000, "ma": 0.24050000000000002, "n_success": 58, "policy_version": 108, "rl_env_steps": 863099, "round": 108, "seed_start": 960000, "sr": 0.29, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 860000, "update_index": 324000}
|
| 45 |
+
{"chunk_transitions": 111000, "episodes": 200, "eval_index": 44, "gain": 0.010499999999999982, "gradient_updates": 333000, "ma": 0.2475, "n_success": 54, "policy_version": 111, "rl_env_steps": 887074, "round": 111, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 880000, "update_index": 333000}
|
| 46 |
+
{"chunk_transitions": 113000, "episodes": 200, "eval_index": 45, "gain": 0.0025000000000000022, "gradient_updates": 339000, "ma": 0.23900000000000002, "n_success": 33, "policy_version": 113, "rl_env_steps": 903052, "round": 113, "seed_start": 960000, "sr": 0.165, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 900000, "update_index": 339000}
|
| 47 |
+
{"chunk_transitions": 116000, "episodes": 200, "eval_index": 46, "gain": -0.00849999999999998, "gradient_updates": 348000, "ma": 0.233, "n_success": 39, "policy_version": 116, "rl_env_steps": 927021, "round": 116, "seed_start": 960000, "sr": 0.195, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 920000, "update_index": 348000}
|
| 48 |
+
{"chunk_transitions": 118000, "episodes": 200, "eval_index": 47, "gain": -0.017500000000000016, "gradient_updates": 354000, "ma": 0.223, "n_success": 37, "policy_version": 118, "rl_env_steps": 943005, "round": 118, "seed_start": 960000, "sr": 0.185, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 940000, "update_index": 354000}
|
| 49 |
+
{"chunk_transitions": 121000, "episodes": 200, "eval_index": 48, "gain": -0.006500000000000006, "gradient_updates": 363000, "ma": 0.22999999999999998, "n_success": 52, "policy_version": 121, "rl_env_steps": 966968, "round": 121, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 960000, "update_index": 363000}
|
| 50 |
+
{"chunk_transitions": 123000, "episodes": 200, "eval_index": 49, "gain": -0.00849999999999998, "gradient_updates": 369000, "ma": 0.23000000000000004, "n_success": 54, "policy_version": 123, "rl_env_steps": 982951, "round": 123, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 980000, "update_index": 369000}
|
| 51 |
+
{"chunk_transitions": 126000, "episodes": 200, "eval_index": 50, "gain": -0.011499999999999955, "gradient_updates": 378000, "ma": 0.23000000000000004, "n_success": 49, "policy_version": 126, "rl_env_steps": 1006914, "round": 126, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1000000, "update_index": 378000}
|
| 52 |
+
{"chunk_transitions": 128000, "episodes": 200, "eval_index": 51, "gain": -0.0010000000000000286, "gradient_updates": 384000, "ma": 0.23349999999999999, "n_success": 51, "policy_version": 128, "rl_env_steps": 1022908, "round": 128, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1020000, "update_index": 384000}
|
| 53 |
+
{"chunk_transitions": 131000, "episodes": 200, "eval_index": 52, "gain": 0.0024999999999999467, "gradient_updates": 393000, "ma": 0.2405, "n_success": 54, "policy_version": 131, "rl_env_steps": 1046888, "round": 131, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1040000, "update_index": 393000}
|
| 54 |
+
{"chunk_transitions": 133000, "episodes": 200, "eval_index": 53, "gain": -0.0009999999999999731, "gradient_updates": 399000, "ma": 0.23950000000000005, "n_success": 56, "policy_version": 133, "rl_env_steps": 1062868, "round": 133, "seed_start": 960000, "sr": 0.28, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1060000, "update_index": 399000}
|
| 55 |
+
{"chunk_transitions": 136000, "episodes": 200, "eval_index": 54, "gain": -0.010999999999999982, "gradient_updates": 408000, "ma": 0.23650000000000002, "n_success": 48, "policy_version": 136, "rl_env_steps": 1086846, "round": 136, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1080000, "update_index": 408000}
|
| 56 |
+
{"chunk_transitions": 138000, "episodes": 200, "eval_index": 55, "gain": 0.010000000000000009, "gradient_updates": 414000, "ma": 0.24900000000000003, "n_success": 58, "policy_version": 138, "rl_env_steps": 1102833, "round": 138, "seed_start": 960000, "sr": 0.29, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1100000, "update_index": 414000}
|
| 57 |
+
{"chunk_transitions": 141000, "episodes": 200, "eval_index": 56, "gain": 0.017499999999999988, "gradient_updates": 423000, "ma": 0.2505, "n_success": 42, "policy_version": 141, "rl_env_steps": 1126809, "round": 141, "seed_start": 960000, "sr": 0.21, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1120000, "update_index": 423000}
|
| 58 |
+
{"chunk_transitions": 143000, "episodes": 200, "eval_index": 57, "gain": 0.03350000000000006, "gradient_updates": 429000, "ma": 0.25650000000000006, "n_success": 49, "policy_version": 143, "rl_env_steps": 1142797, "round": 143, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1140000, "update_index": 429000}
|
| 59 |
+
{"chunk_transitions": 146000, "episodes": 200, "eval_index": 58, "gain": 0.025000000000000022, "gradient_updates": 438000, "ma": 0.255, "n_success": 49, "policy_version": 146, "rl_env_steps": 1166783, "round": 146, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1160000, "update_index": 438000}
|
| 60 |
+
{"chunk_transitions": 148000, "episodes": 200, "eval_index": 59, "gain": 0.020999999999999963, "gradient_updates": 444000, "ma": 0.251, "n_success": 46, "policy_version": 148, "rl_env_steps": 1182767, "round": 148, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1180000, "update_index": 444000}
|
| 61 |
+
{"chunk_transitions": 151000, "episodes": 200, "eval_index": 60, "gain": 0.01849999999999999, "gradient_updates": 453000, "ma": 0.24850000000000003, "n_success": 44, "policy_version": 151, "rl_env_steps": 1206758, "round": 151, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1200000, "update_index": 453000}
|
| 62 |
+
{"chunk_transitions": 153000, "episodes": 200, "eval_index": 61, "gain": 0.010000000000000009, "gradient_updates": 459000, "ma": 0.2435, "n_success": 41, "policy_version": 153, "rl_env_steps": 1222750, "round": 153, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1220000, "update_index": 459000}
|
| 63 |
+
{"chunk_transitions": 156000, "episodes": 200, "eval_index": 62, "gain": 0.0020000000000000018, "gradient_updates": 468000, "ma": 0.2425, "n_success": 52, "policy_version": 156, "rl_env_steps": 1246744, "round": 156, "seed_start": 960000, "sr": 0.26, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1240000, "update_index": 468000}
|
| 64 |
+
{"chunk_transitions": 158000, "episodes": 200, "eval_index": 63, "gain": -0.0010000000000000564, "gradient_updates": 474000, "ma": 0.2385, "n_success": 48, "policy_version": 158, "rl_env_steps": 1262713, "round": 158, "seed_start": 960000, "sr": 0.24, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1260000, "update_index": 474000}
|
| 65 |
+
{"chunk_transitions": 161000, "episodes": 200, "eval_index": 64, "gain": -0.0015000000000000013, "gradient_updates": 483000, "ma": 0.23500000000000001, "n_success": 41, "policy_version": 161, "rl_env_steps": 1286701, "round": 161, "seed_start": 960000, "sr": 0.205, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1280000, "update_index": 483000}
|
| 66 |
+
{"chunk_transitions": 163000, "episodes": 200, "eval_index": 65, "gain": -0.018500000000000016, "gradient_updates": 489000, "ma": 0.2305, "n_success": 49, "policy_version": 163, "rl_env_steps": 1302679, "round": 163, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1300000, "update_index": 489000}
|
| 67 |
+
{"chunk_transitions": 166000, "episodes": 200, "eval_index": 66, "gain": -0.01350000000000004, "gradient_updates": 498000, "ma": 0.23699999999999996, "n_success": 55, "policy_version": 166, "rl_env_steps": 1326658, "round": 166, "seed_start": 960000, "sr": 0.275, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1320000, "update_index": 498000}
|
| 68 |
+
{"chunk_transitions": 168000, "episodes": 200, "eval_index": 67, "gain": -0.021000000000000074, "gradient_updates": 504000, "ma": 0.2355, "n_success": 46, "policy_version": 168, "rl_env_steps": 1342626, "round": 168, "seed_start": 960000, "sr": 0.23, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1340000, "update_index": 504000}
|
| 69 |
+
{"chunk_transitions": 171000, "episodes": 200, "eval_index": 68, "gain": -0.017000000000000015, "gradient_updates": 513000, "ma": 0.238, "n_success": 54, "policy_version": 171, "rl_env_steps": 1366615, "round": 171, "seed_start": 960000, "sr": 0.27, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1360000, "update_index": 513000}
|
| 70 |
+
{"chunk_transitions": 173000, "episodes": 200, "eval_index": 69, "gain": -0.018500000000000016, "gradient_updates": 519000, "ma": 0.23249999999999998, "n_success": 35, "policy_version": 173, "rl_env_steps": 1382606, "round": 173, "seed_start": 960000, "sr": 0.175, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1380000, "update_index": 519000}
|
| 71 |
+
{"chunk_transitions": 176000, "episodes": 200, "eval_index": 70, "gain": -0.012500000000000039, "gradient_updates": 528000, "ma": 0.236, "n_success": 51, "policy_version": 176, "rl_env_steps": 1406584, "round": 176, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1400000, "update_index": 528000}
|
| 72 |
+
{"chunk_transitions": 178000, "episodes": 200, "eval_index": 71, "gain": -0.012500000000000039, "gradient_updates": 534000, "ma": 0.23099999999999996, "n_success": 31, "policy_version": 178, "rl_env_steps": 1422552, "round": 178, "seed_start": 960000, "sr": 0.155, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1420000, "update_index": 534000}
|
| 73 |
+
{"chunk_transitions": 181000, "episodes": 200, "eval_index": 72, "gain": -0.015000000000000013, "gradient_updates": 543000, "ma": 0.22749999999999998, "n_success": 45, "policy_version": 181, "rl_env_steps": 1446534, "round": 181, "seed_start": 960000, "sr": 0.225, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1440000, "update_index": 543000}
|
| 74 |
+
{"chunk_transitions": 183000, "episodes": 200, "eval_index": 73, "gain": -0.00949999999999998, "gradient_updates": 549000, "ma": 0.229, "n_success": 51, "policy_version": 183, "rl_env_steps": 1462512, "round": 183, "seed_start": 960000, "sr": 0.255, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1460000, "update_index": 549000}
|
| 75 |
+
{"chunk_transitions": 186000, "episodes": 200, "eval_index": 74, "gain": -0.004500000000000004, "gradient_updates": 558000, "ma": 0.2305, "n_success": 44, "policy_version": 186, "rl_env_steps": 1486479, "round": 186, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1480000, "update_index": 558000}
|
| 76 |
+
{"chunk_transitions": 188000, "episodes": 200, "eval_index": 75, "gain": -0.005000000000000032, "gradient_updates": 564000, "ma": 0.22549999999999998, "n_success": 39, "policy_version": 188, "rl_env_steps": 1502465, "round": 188, "seed_start": 960000, "sr": 0.195, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1500000, "update_index": 564000}
|
| 77 |
+
{"chunk_transitions": 191000, "episodes": 200, "eval_index": 76, "gain": -0.00899999999999998, "gradient_updates": 573000, "ma": 0.22799999999999998, "n_success": 60, "policy_version": 191, "rl_env_steps": 1526448, "round": 191, "seed_start": 960000, "sr": 0.3, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1520000, "update_index": 573000}
|
| 78 |
+
{"chunk_transitions": 193000, "episodes": 200, "eval_index": 77, "gain": -0.007000000000000006, "gradient_updates": 579000, "ma": 0.22849999999999998, "n_success": 47, "policy_version": 193, "rl_env_steps": 1542429, "round": 193, "seed_start": 960000, "sr": 0.235, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1540000, "update_index": 579000}
|
| 79 |
+
{"chunk_transitions": 196000, "episodes": 200, "eval_index": 78, "gain": -0.011499999999999982, "gradient_updates": 588000, "ma": 0.2265, "n_success": 50, "policy_version": 196, "rl_env_steps": 1566419, "round": 196, "seed_start": 960000, "sr": 0.25, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1560000, "update_index": 588000}
|
| 80 |
+
{"chunk_transitions": 198000, "episodes": 200, "eval_index": 79, "gain": 0.0010000000000000009, "gradient_updates": 594000, "ma": 0.23349999999999999, "n_success": 49, "policy_version": 198, "rl_env_steps": 1582406, "round": 198, "seed_start": 960000, "sr": 0.245, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1580000, "update_index": 594000}
|
| 81 |
+
{"chunk_transitions": 201000, "episodes": 200, "eval_index": 80, "gain": -0.00599999999999995, "gradient_updates": 603000, "ma": 0.23000000000000004, "n_success": 44, "policy_version": 201, "rl_env_steps": 1606374, "round": 201, "seed_start": 960000, "sr": 0.22, "success_requested": 0, "success_requested_imitation": 0, "success_rounds_with_skips": 0, "success_skipped": 0, "success_skipped_imitation": 0, "success_skipped_td": 0, "target_step": 1600000, "update_index": 603000}
|
RL/ToolHang100/QC/diverse100/seed_1/eval/final.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1bd80e3bf7d314df54e9e1acf47902ada35684961b3ef9f28271851aec8e3067
|
| 3 |
+
size 6810
|
RL/ToolHang100/QC/diverse100/seed_1/final/source_metadata.json
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"checkpoint_sha256": "8f507af6fb47652d3b5b512d06c8bc51d2da9d2bab494670f88f9b9ce859104d",
|
| 3 |
+
"evaluation": {
|
| 4 |
+
"episodes": 300,
|
| 5 |
+
"imputed": false,
|
| 6 |
+
"n_success": 77,
|
| 7 |
+
"observed": true,
|
| 8 |
+
"seed_start": 800000,
|
| 9 |
+
"sr": 0.25666666666666665,
|
| 10 |
+
"successes": [
|
| 11 |
+
0,
|
| 12 |
+
1,
|
| 13 |
+
0,
|
| 14 |
+
1,
|
| 15 |
+
0,
|
| 16 |
+
0,
|
| 17 |
+
0,
|
| 18 |
+
0,
|
| 19 |
+
1,
|
| 20 |
+
0,
|
| 21 |
+
0,
|
| 22 |
+
0,
|
| 23 |
+
0,
|
| 24 |
+
1,
|
| 25 |
+
1,
|
| 26 |
+
0,
|
| 27 |
+
0,
|
| 28 |
+
1,
|
| 29 |
+
0,
|
| 30 |
+
0,
|
| 31 |
+
0,
|
| 32 |
+
0,
|
| 33 |
+
1,
|
| 34 |
+
0,
|
| 35 |
+
1,
|
| 36 |
+
0,
|
| 37 |
+
0,
|
| 38 |
+
0,
|
| 39 |
+
1,
|
| 40 |
+
0,
|
| 41 |
+
0,
|
| 42 |
+
0,
|
| 43 |
+
0,
|
| 44 |
+
0,
|
| 45 |
+
0,
|
| 46 |
+
0,
|
| 47 |
+
0,
|
| 48 |
+
0,
|
| 49 |
+
0,
|
| 50 |
+
0,
|
| 51 |
+
1,
|
| 52 |
+
0,
|
| 53 |
+
0,
|
| 54 |
+
1,
|
| 55 |
+
0,
|
| 56 |
+
0,
|
| 57 |
+
0,
|
| 58 |
+
0,
|
| 59 |
+
0,
|
| 60 |
+
0,
|
| 61 |
+
0,
|
| 62 |
+
0,
|
| 63 |
+
0,
|
| 64 |
+
0,
|
| 65 |
+
0,
|
| 66 |
+
0,
|
| 67 |
+
0,
|
| 68 |
+
0,
|
| 69 |
+
0,
|
| 70 |
+
0,
|
| 71 |
+
0,
|
| 72 |
+
0,
|
| 73 |
+
0,
|
| 74 |
+
1,
|
| 75 |
+
1,
|
| 76 |
+
1,
|
| 77 |
+
0,
|
| 78 |
+
0,
|
| 79 |
+
0,
|
| 80 |
+
1,
|
| 81 |
+
0,
|
| 82 |
+
1,
|
| 83 |
+
0,
|
| 84 |
+
0,
|
| 85 |
+
0,
|
| 86 |
+
0,
|
| 87 |
+
0,
|
| 88 |
+
0,
|
| 89 |
+
0,
|
| 90 |
+
1,
|
| 91 |
+
0,
|
| 92 |
+
0,
|
| 93 |
+
0,
|
| 94 |
+
0,
|
| 95 |
+
0,
|
| 96 |
+
0,
|
| 97 |
+
0,
|
| 98 |
+
0,
|
| 99 |
+
0,
|
| 100 |
+
0,
|
| 101 |
+
0,
|
| 102 |
+
0,
|
| 103 |
+
0,
|
| 104 |
+
1,
|
| 105 |
+
1,
|
| 106 |
+
0,
|
| 107 |
+
0,
|
| 108 |
+
0,
|
| 109 |
+
0,
|
| 110 |
+
0,
|
| 111 |
+
0,
|
| 112 |
+
0,
|
| 113 |
+
1,
|
| 114 |
+
0,
|
| 115 |
+
1,
|
| 116 |
+
1,
|
| 117 |
+
0,
|
| 118 |
+
0,
|
| 119 |
+
1,
|
| 120 |
+
0,
|
| 121 |
+
0,
|
| 122 |
+
0,
|
| 123 |
+
0,
|
| 124 |
+
0,
|
| 125 |
+
0,
|
| 126 |
+
0,
|
| 127 |
+
1,
|
| 128 |
+
0,
|
| 129 |
+
0,
|
| 130 |
+
0,
|
| 131 |
+
0,
|
| 132 |
+
0,
|
| 133 |
+
1,
|
| 134 |
+
0,
|
| 135 |
+
0,
|
| 136 |
+
0,
|
| 137 |
+
0,
|
| 138 |
+
0,
|
| 139 |
+
0,
|
| 140 |
+
0,
|
| 141 |
+
0,
|
| 142 |
+
0,
|
| 143 |
+
0,
|
| 144 |
+
0,
|
| 145 |
+
1,
|
| 146 |
+
0,
|
| 147 |
+
0,
|
| 148 |
+
0,
|
| 149 |
+
0,
|
| 150 |
+
0,
|
| 151 |
+
1,
|
| 152 |
+
0,
|
| 153 |
+
0,
|
| 154 |
+
0,
|
| 155 |
+
0,
|
| 156 |
+
0,
|
| 157 |
+
0,
|
| 158 |
+
0,
|
| 159 |
+
1,
|
| 160 |
+
0,
|
| 161 |
+
0,
|
| 162 |
+
1,
|
| 163 |
+
0,
|
| 164 |
+
0,
|
| 165 |
+
1,
|
| 166 |
+
0,
|
| 167 |
+
0,
|
| 168 |
+
0,
|
| 169 |
+
0,
|
| 170 |
+
0,
|
| 171 |
+
1,
|
| 172 |
+
1,
|
| 173 |
+
0,
|
| 174 |
+
1,
|
| 175 |
+
0,
|
| 176 |
+
0,
|
| 177 |
+
0,
|
| 178 |
+
0,
|
| 179 |
+
1,
|
| 180 |
+
0,
|
| 181 |
+
1,
|
| 182 |
+
1,
|
| 183 |
+
0,
|
| 184 |
+
0,
|
| 185 |
+
0,
|
| 186 |
+
0,
|
| 187 |
+
0,
|
| 188 |
+
0,
|
| 189 |
+
0,
|
| 190 |
+
1,
|
| 191 |
+
0,
|
| 192 |
+
1,
|
| 193 |
+
0,
|
| 194 |
+
0,
|
| 195 |
+
0,
|
| 196 |
+
1,
|
| 197 |
+
1,
|
| 198 |
+
0,
|
| 199 |
+
1,
|
| 200 |
+
0,
|
| 201 |
+
0,
|
| 202 |
+
0,
|
| 203 |
+
0,
|
| 204 |
+
0,
|
| 205 |
+
0,
|
| 206 |
+
1,
|
| 207 |
+
1,
|
| 208 |
+
1,
|
| 209 |
+
0,
|
| 210 |
+
0,
|
| 211 |
+
1,
|
| 212 |
+
0,
|
| 213 |
+
0,
|
| 214 |
+
0,
|
| 215 |
+
0,
|
| 216 |
+
1,
|
| 217 |
+
1,
|
| 218 |
+
0,
|
| 219 |
+
0,
|
| 220 |
+
0,
|
| 221 |
+
0,
|
| 222 |
+
1,
|
| 223 |
+
1,
|
| 224 |
+
0,
|
| 225 |
+
0,
|
| 226 |
+
1,
|
| 227 |
+
1,
|
| 228 |
+
0,
|
| 229 |
+
0,
|
| 230 |
+
0,
|
| 231 |
+
0,
|
| 232 |
+
0,
|
| 233 |
+
0,
|
| 234 |
+
0,
|
| 235 |
+
0,
|
| 236 |
+
1,
|
| 237 |
+
0,
|
| 238 |
+
0,
|
| 239 |
+
1,
|
| 240 |
+
0,
|
| 241 |
+
1,
|
| 242 |
+
1,
|
| 243 |
+
0,
|
| 244 |
+
0,
|
| 245 |
+
0,
|
| 246 |
+
1,
|
| 247 |
+
0,
|
| 248 |
+
0,
|
| 249 |
+
0,
|
| 250 |
+
0,
|
| 251 |
+
0,
|
| 252 |
+
0,
|
| 253 |
+
1,
|
| 254 |
+
1,
|
| 255 |
+
1,
|
| 256 |
+
0,
|
| 257 |
+
0,
|
| 258 |
+
0,
|
| 259 |
+
0,
|
| 260 |
+
0,
|
| 261 |
+
0,
|
| 262 |
+
0,
|
| 263 |
+
0,
|
| 264 |
+
1,
|
| 265 |
+
0,
|
| 266 |
+
1,
|
| 267 |
+
1,
|
| 268 |
+
0,
|
| 269 |
+
0,
|
| 270 |
+
1,
|
| 271 |
+
0,
|
| 272 |
+
1,
|
| 273 |
+
1,
|
| 274 |
+
0,
|
| 275 |
+
0,
|
| 276 |
+
0,
|
| 277 |
+
1,
|
| 278 |
+
0,
|
| 279 |
+
0,
|
| 280 |
+
0,
|
| 281 |
+
0,
|
| 282 |
+
1,
|
| 283 |
+
0,
|
| 284 |
+
0,
|
| 285 |
+
1,
|
| 286 |
+
1,
|
| 287 |
+
0,
|
| 288 |
+
0,
|
| 289 |
+
0,
|
| 290 |
+
0,
|
| 291 |
+
0,
|
| 292 |
+
0,
|
| 293 |
+
0,
|
| 294 |
+
1,
|
| 295 |
+
1,
|
| 296 |
+
0,
|
| 297 |
+
1,
|
| 298 |
+
0,
|
| 299 |
+
1,
|
| 300 |
+
0,
|
| 301 |
+
0,
|
| 302 |
+
1,
|
| 303 |
+
1,
|
| 304 |
+
0,
|
| 305 |
+
0,
|
| 306 |
+
0,
|
| 307 |
+
1,
|
| 308 |
+
1,
|
| 309 |
+
0,
|
| 310 |
+
0
|
| 311 |
+
]
|
| 312 |
+
},
|
| 313 |
+
"status": "budget_complete"
|
| 314 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/final/source_resolved_config.json
ADDED
|
@@ -0,0 +1,1195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
|
| 3 |
+
"env": {
|
| 4 |
+
"chunk_length": 8,
|
| 5 |
+
"n_envs": 4
|
| 6 |
+
},
|
| 7 |
+
"learner": {
|
| 8 |
+
"convention": "unified_env_step",
|
| 9 |
+
"gamma_env": 0.999
|
| 10 |
+
},
|
| 11 |
+
"overrides": [
|
| 12 |
+
{
|
| 13 |
+
"decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
|
| 14 |
+
"key": "n_step_chunks",
|
| 15 |
+
"reason": "task-specific unified Study2 setting",
|
| 16 |
+
"upstream": 3,
|
| 17 |
+
"value": 5
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 21 |
+
"key": "offline.cohort_size",
|
| 22 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 23 |
+
"upstream": 50,
|
| 24 |
+
"value": 100
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 28 |
+
"key": "offline.episodes",
|
| 29 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 30 |
+
"upstream": 50,
|
| 31 |
+
"value": 100
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"decision_ref": "[0930-S2DEC]",
|
| 35 |
+
"key": "learner.qc.agent.act_bounds_json",
|
| 36 |
+
"reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
|
| 37 |
+
"upstream": null,
|
| 38 |
+
"value": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"decision_ref": "[0930-S2DEC]",
|
| 42 |
+
"key": "learner.qc.agent.act_max",
|
| 43 |
+
"reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
|
| 44 |
+
"upstream": 1.0,
|
| 45 |
+
"value": 3.0058202743530273
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"decision_ref": "[0930-S2DEC]",
|
| 49 |
+
"key": "learner.qc.agent.act_min",
|
| 50 |
+
"reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
|
| 51 |
+
"upstream": -1.0,
|
| 52 |
+
"value": -2.9472272396087646
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"decision_ref": "[0930-S2DEC]",
|
| 56 |
+
"key": "learner.qc.agent.actor_lr_partition",
|
| 57 |
+
"reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
|
| 58 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 59 |
+
"value": 0.0001
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 63 |
+
"key": "learner.qc.agent.actor_num_samples",
|
| 64 |
+
"reason": "single BoN knob including QC TD",
|
| 65 |
+
"upstream": 32,
|
| 66 |
+
"value": 16
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 70 |
+
"key": "learner.qc.agent.best_of_n",
|
| 71 |
+
"reason": "single BoN knob: collection/eval/QC TD",
|
| 72 |
+
"upstream": 8,
|
| 73 |
+
"value": 16
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 77 |
+
"key": "learner.qc.agent.bon_agg",
|
| 78 |
+
"reason": "Best-of-N selection by the ensemble mean",
|
| 79 |
+
"upstream": null,
|
| 80 |
+
"value": "mean"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 84 |
+
"key": "learner.qc.agent.bon_sft",
|
| 85 |
+
"reason": "variant qc flag bon_sft",
|
| 86 |
+
"upstream": "upstream_yaml",
|
| 87 |
+
"value": false
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"decision_ref": "[0930-S2DEC]",
|
| 91 |
+
"key": "learner.qc.agent.critic_lr_partition",
|
| 92 |
+
"reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
|
| 93 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 94 |
+
"value": 0.0003
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 98 |
+
"key": "learner.qc.agent.partition_optimizer",
|
| 99 |
+
"reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
|
| 100 |
+
"upstream": null,
|
| 101 |
+
"value": true
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 105 |
+
"key": "learner.qc.agent.q_target_agg",
|
| 106 |
+
"reason": "REDQ-style TD target: min over a random critic subset",
|
| 107 |
+
"upstream": null,
|
| 108 |
+
"value": "subset_min"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 112 |
+
"key": "learner.qc.agent.q_target_replace",
|
| 113 |
+
"reason": "subset drawn without replacement",
|
| 114 |
+
"upstream": null,
|
| 115 |
+
"value": false
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 119 |
+
"key": "learner.qc.agent.q_target_subset",
|
| 120 |
+
"reason": "TD-target subset size",
|
| 121 |
+
"upstream": null,
|
| 122 |
+
"value": 2
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"decision_ref": "[0930-S2DEC]",
|
| 126 |
+
"key": "learner.qc.agent.tau",
|
| 127 |
+
"reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
|
| 128 |
+
"upstream": 0.05,
|
| 129 |
+
"value": 0.005
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"decision_ref": "[0930-S2DEC]",
|
| 133 |
+
"key": "learner.qc.agent.value_hidden_dims",
|
| 134 |
+
"reason": "unified.critic_hidden (critics only; actors native)",
|
| 135 |
+
"upstream": [
|
| 136 |
+
512,
|
| 137 |
+
512,
|
| 138 |
+
512,
|
| 139 |
+
512,
|
| 140 |
+
512
|
| 141 |
+
],
|
| 142 |
+
"value": [
|
| 143 |
+
512,
|
| 144 |
+
512,
|
| 145 |
+
512
|
| 146 |
+
]
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 150 |
+
"key": "learner.qc.training.bc_pi_steps",
|
| 151 |
+
"reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
|
| 152 |
+
"upstream": 1000000,
|
| 153 |
+
"value": 0
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 157 |
+
"key": "learner.qc.loop.buffer.utd_online",
|
| 158 |
+
"reason": "UTD=3 per executed action chunk",
|
| 159 |
+
"upstream": 1,
|
| 160 |
+
"value": "3 gradient updates per executed chunk"
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 164 |
+
"key": "learner.qc.loop.n_step",
|
| 165 |
+
"reason": "n-step=3 counted in action chunks",
|
| 166 |
+
"upstream": "1 chunk (sequence_length = horizon_length)",
|
| 167 |
+
"value": "5 executed chunks (unified_env_step window)"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 171 |
+
"key": "learner.qc.loop.critic_actor_ratio",
|
| 172 |
+
"reason": "critic every update, actor every 2nd global update",
|
| 173 |
+
"upstream": "1:1 (utd_q = utd_pi)",
|
| 174 |
+
"value": "2:1"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 178 |
+
"key": "learner.qc.loop.buffer.start_training",
|
| 179 |
+
"reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
|
| 180 |
+
"upstream": 10000,
|
| 181 |
+
"value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 185 |
+
"key": "learner.qc.loop.buffer.offline_ratio",
|
| 186 |
+
"reason": "[1017-S2OFFBC] adaptive_ratio off",
|
| 187 |
+
"upstream": 0.0,
|
| 188 |
+
"value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 192 |
+
"key": "learner.qc.loop.buffer.size",
|
| 193 |
+
"reason": "one shared replay of executed chunks; success buffer = mask",
|
| 194 |
+
"upstream": 2000000,
|
| 195 |
+
"value": "1000000 executed chunks, shared FIFO"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 199 |
+
"key": "learner.qc.actor_architecture",
|
| 200 |
+
"reason": "Study2 fine-tunes the Study1 pixel_state CFM",
|
| 201 |
+
"upstream": [
|
| 202 |
+
512,
|
| 203 |
+
512,
|
| 204 |
+
512,
|
| 205 |
+
512,
|
| 206 |
+
512
|
| 207 |
+
],
|
| 208 |
+
"value": "checkpoint StandardGenerator (EMA generator copy)"
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 212 |
+
"key": "learner.qc.observation",
|
| 213 |
+
"reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
|
| 214 |
+
"upstream": "low-dim state",
|
| 215 |
+
"value": "frozen Ri features c [137] from the checkpoint encoder"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"decision_ref": "[0930-S2DEC]",
|
| 219 |
+
"key": "learner.qc.reward",
|
| 220 |
+
"reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
|
| 221 |
+
"upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
|
| 222 |
+
"value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"decision_ref": "[0930-S2DEC]",
|
| 226 |
+
"key": "learner.qc.td_target",
|
| 227 |
+
"reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
|
| 228 |
+
"upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
|
| 229 |
+
"value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"decision_ref": "[0930-S2DEC]",
|
| 233 |
+
"key": "learner.qc.eval",
|
| 234 |
+
"reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
|
| 235 |
+
"upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
|
| 236 |
+
"value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 240 |
+
"key": "learner.qc.lr_schedule",
|
| 241 |
+
"reason": "RL env-step clock shared with extra success TD",
|
| 242 |
+
"upstream": "native per-optimizer schedule",
|
| 243 |
+
"value": {
|
| 244 |
+
"actor": 0.0001,
|
| 245 |
+
"critic": 0.0003,
|
| 246 |
+
"decay_end_env_steps": 1606196,
|
| 247 |
+
"min_ratio": 0.1,
|
| 248 |
+
"schedule": "warmup_cosine",
|
| 249 |
+
"temperature": 0.0003,
|
| 250 |
+
"warmup_env_steps": 20000
|
| 251 |
+
}
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"decision_ref": "[0930-S2DEC]",
|
| 255 |
+
"key": "learner.qc.agent.q_min_q_max",
|
| 256 |
+
"reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
|
| 257 |
+
"upstream": [
|
| 258 |
+
-999.9999999999991,
|
| 259 |
+
0.0
|
| 260 |
+
],
|
| 261 |
+
"value": [
|
| 262 |
+
0.0,
|
| 263 |
+
1.0
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
],
|
| 267 |
+
"protocol": {
|
| 268 |
+
"checkpoint_every_rounds": 5,
|
| 269 |
+
"checkpoint_retention": {
|
| 270 |
+
"best": true,
|
| 271 |
+
"collapse_drop": {
|
| 272 |
+
"min_abs": 0.2,
|
| 273 |
+
"ratio": 0.5
|
| 274 |
+
},
|
| 275 |
+
"diagnostics": true,
|
| 276 |
+
"every_evals": 3
|
| 277 |
+
},
|
| 278 |
+
"collection": {
|
| 279 |
+
"n_envs": 4,
|
| 280 |
+
"train_seed_base": 100000000,
|
| 281 |
+
"train_seed_stride": 1000000,
|
| 282 |
+
"training_freq_chunks": 1000
|
| 283 |
+
},
|
| 284 |
+
"convergence": {
|
| 285 |
+
"early_stop": false,
|
| 286 |
+
"gain_lag": 10,
|
| 287 |
+
"high_sr": 0.95,
|
| 288 |
+
"high_sr_evals": 5,
|
| 289 |
+
"ma_window": 10,
|
| 290 |
+
"max_env_steps": 1606196,
|
| 291 |
+
"min_env_steps": 200000,
|
| 292 |
+
"min_gain": 0.02,
|
| 293 |
+
"zero_success_evals": 8
|
| 294 |
+
},
|
| 295 |
+
"critic_ensemble": {
|
| 296 |
+
"num": 10,
|
| 297 |
+
"policy": "mean",
|
| 298 |
+
"target": {
|
| 299 |
+
"kind": "subset_min",
|
| 300 |
+
"replace": false,
|
| 301 |
+
"size": 2
|
| 302 |
+
}
|
| 303 |
+
},
|
| 304 |
+
"eval": {
|
| 305 |
+
"best_of_n": 16,
|
| 306 |
+
"episodes": 200,
|
| 307 |
+
"every_env_steps": 20000,
|
| 308 |
+
"include_step0": true,
|
| 309 |
+
"seed_start": 960000,
|
| 310 |
+
"workers": 4
|
| 311 |
+
},
|
| 312 |
+
"final_eval": {
|
| 313 |
+
"episodes": 300,
|
| 314 |
+
"on_zero_stop": true,
|
| 315 |
+
"seed_start": 800000
|
| 316 |
+
},
|
| 317 |
+
"n_step_chunks": 5,
|
| 318 |
+
"offline": {
|
| 319 |
+
"adaptive_ratio": false,
|
| 320 |
+
"cohort_size": 100,
|
| 321 |
+
"episodes": 100,
|
| 322 |
+
"learners_without_offline": [
|
| 323 |
+
"dsrl_sac"
|
| 324 |
+
],
|
| 325 |
+
"ratio": {
|
| 326 |
+
"decay_env_steps": {
|
| 327 |
+
"can": 160000,
|
| 328 |
+
"square": 320000,
|
| 329 |
+
"tool_hang": 640000,
|
| 330 |
+
"transport": 320000
|
| 331 |
+
},
|
| 332 |
+
"end": 0.1,
|
| 333 |
+
"start": {
|
| 334 |
+
"can": 0.7,
|
| 335 |
+
"square": 0.5,
|
| 336 |
+
"tool_hang": 0.5,
|
| 337 |
+
"transport": 0.9
|
| 338 |
+
}
|
| 339 |
+
},
|
| 340 |
+
"selection": "ascending_demo_index",
|
| 341 |
+
"source": "bc_regime",
|
| 342 |
+
"unsupported_learners": [
|
| 343 |
+
"zprl"
|
| 344 |
+
]
|
| 345 |
+
},
|
| 346 |
+
"replay": {
|
| 347 |
+
"capacity_chunks": 1000000,
|
| 348 |
+
"pin_start_successes": false,
|
| 349 |
+
"success_buffer": "success_mask_on_shared_fifo",
|
| 350 |
+
"success_skip_rule": "skip_below_success_batch_size",
|
| 351 |
+
"unit": "executed_chunk"
|
| 352 |
+
},
|
| 353 |
+
"start": {
|
| 354 |
+
"max_episodes": 200,
|
| 355 |
+
"mode": "offline_buffer",
|
| 356 |
+
"policy": {
|
| 357 |
+
"default": "base_cfm_ode",
|
| 358 |
+
"zprl": "vib_base"
|
| 359 |
+
},
|
| 360 |
+
"target_successes": 20
|
| 361 |
+
},
|
| 362 |
+
"unified": {
|
| 363 |
+
"actor_target_tau": 0.005,
|
| 364 |
+
"critic_hidden": [
|
| 365 |
+
512,
|
| 366 |
+
512,
|
| 367 |
+
512
|
| 368 |
+
],
|
| 369 |
+
"critic_target_every_critic_step": true,
|
| 370 |
+
"critic_target_tau": 0.005,
|
| 371 |
+
"dsrl_noise_imitation_reference_dims": 28,
|
| 372 |
+
"gamma_env": 0.999,
|
| 373 |
+
"lr": {
|
| 374 |
+
"actor": 0.0001,
|
| 375 |
+
"critic": 0.0003,
|
| 376 |
+
"decay_end_env_steps": 1606196,
|
| 377 |
+
"min_ratio": 0.1,
|
| 378 |
+
"schedule": "warmup_cosine",
|
| 379 |
+
"temperature": 0.0003,
|
| 380 |
+
"warmup_env_steps": 20000
|
| 381 |
+
},
|
| 382 |
+
"reward": "binary_first_success",
|
| 383 |
+
"success_td_polyak": true,
|
| 384 |
+
"timeout_bootstrap": false,
|
| 385 |
+
"zprl_imitation": "mse_pretanh_mean"
|
| 386 |
+
},
|
| 387 |
+
"updates": {
|
| 388 |
+
"batch_size": 256,
|
| 389 |
+
"critic_per_actor": 2,
|
| 390 |
+
"success_batch_size": 256,
|
| 391 |
+
"utd_per_chunk": 3
|
| 392 |
+
},
|
| 393 |
+
"version": "study2_protocol_v10"
|
| 394 |
+
},
|
| 395 |
+
"protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
|
| 396 |
+
"provenance_extra": {
|
| 397 |
+
"agent_library": {
|
| 398 |
+
"framework": "jax",
|
| 399 |
+
"sources": {
|
| 400 |
+
"__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
|
| 401 |
+
"_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 402 |
+
"_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 403 |
+
"_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 404 |
+
"_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 405 |
+
"_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
|
| 406 |
+
"_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
|
| 407 |
+
"_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
|
| 408 |
+
"_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
|
| 409 |
+
"_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
|
| 410 |
+
"_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
|
| 411 |
+
"_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
|
| 412 |
+
"_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
|
| 413 |
+
"_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
|
| 414 |
+
"_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
|
| 415 |
+
"_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
|
| 416 |
+
"_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
|
| 417 |
+
"_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
|
| 418 |
+
"_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 419 |
+
"_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
|
| 420 |
+
"_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
|
| 421 |
+
"_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
|
| 422 |
+
"_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
|
| 423 |
+
"_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
|
| 424 |
+
"_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
|
| 425 |
+
"_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
|
| 426 |
+
"_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
|
| 427 |
+
"_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
|
| 428 |
+
"_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
|
| 429 |
+
"_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
|
| 430 |
+
"_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
|
| 431 |
+
"dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
|
| 432 |
+
"dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
|
| 433 |
+
"dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
|
| 434 |
+
"factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
|
| 435 |
+
"flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
|
| 436 |
+
"native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
|
| 437 |
+
"native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
|
| 438 |
+
"native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
|
| 439 |
+
"native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
|
| 440 |
+
"native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
|
| 441 |
+
"networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
|
| 442 |
+
"ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
|
| 443 |
+
"optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
|
| 444 |
+
"qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
|
| 445 |
+
"registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
|
| 446 |
+
"residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
|
| 447 |
+
"telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
|
| 448 |
+
"train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
|
| 449 |
+
"vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
|
| 450 |
+
"vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
|
| 451 |
+
"zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
|
| 452 |
+
},
|
| 453 |
+
"version": "study2_jax_agents_v2"
|
| 454 |
+
},
|
| 455 |
+
"base_framework": "jax",
|
| 456 |
+
"bc_identity": {
|
| 457 |
+
"architecture": "dice_vision_resnet_flow_unet_v1",
|
| 458 |
+
"family": "StandardBaseBC",
|
| 459 |
+
"framework": "jax"
|
| 460 |
+
},
|
| 461 |
+
"checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261024_toolhang_diverse100_v10_three/base/diverse100/epoch_300.pkl",
|
| 462 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 463 |
+
"experiment": "tool_hang",
|
| 464 |
+
"frozen_hash": "719a7185a0d6a4877c76a5e1d9b1adb163a3d863d28bced54e79d534e8db6873",
|
| 465 |
+
"learner": {
|
| 466 |
+
"action_bounds": {
|
| 467 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 468 |
+
"compat_act_max": 3.0058202743530273,
|
| 469 |
+
"compat_act_min": -2.9472272396087646,
|
| 470 |
+
"hi": [
|
| 471 |
+
1.0,
|
| 472 |
+
1.0,
|
| 473 |
+
1.0,
|
| 474 |
+
3.0058202743530273,
|
| 475 |
+
1.8893057107925415,
|
| 476 |
+
1.0,
|
| 477 |
+
1.0
|
| 478 |
+
],
|
| 479 |
+
"kind": "per_dimension",
|
| 480 |
+
"lo": [
|
| 481 |
+
-1.0,
|
| 482 |
+
-1.0,
|
| 483 |
+
-1.0,
|
| 484 |
+
-2.9472272396087646,
|
| 485 |
+
-1.9721344709396362,
|
| 486 |
+
-1.318867564201355,
|
| 487 |
+
-1.0
|
| 488 |
+
],
|
| 489 |
+
"note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
|
| 490 |
+
},
|
| 491 |
+
"agent_config": {
|
| 492 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 493 |
+
"act_max": 3.0058202743530273,
|
| 494 |
+
"act_min": -2.9472272396087646,
|
| 495 |
+
"action_chunking": true,
|
| 496 |
+
"action_critic_hidden_dims": null,
|
| 497 |
+
"action_dim": 7,
|
| 498 |
+
"action_num_qs": 10,
|
| 499 |
+
"action_repeat": false,
|
| 500 |
+
"actor_backbone": "mlp",
|
| 501 |
+
"actor_decay_steps": 50000,
|
| 502 |
+
"actor_end_value": 2e-05,
|
| 503 |
+
"actor_hidden_dims": [
|
| 504 |
+
512,
|
| 505 |
+
512,
|
| 506 |
+
512,
|
| 507 |
+
512,
|
| 508 |
+
512
|
| 509 |
+
],
|
| 510 |
+
"actor_layer_norm": false,
|
| 511 |
+
"actor_lr": 0.0003,
|
| 512 |
+
"actor_lr_partition": 0.0001,
|
| 513 |
+
"actor_num_samples": 16,
|
| 514 |
+
"actor_obs": "state",
|
| 515 |
+
"actor_scheduler": "cosine",
|
| 516 |
+
"actor_tau": 0.005,
|
| 517 |
+
"actor_type": "best-of-n",
|
| 518 |
+
"actor_warmup_steps": 2000,
|
| 519 |
+
"actor_weight_decay": 0.0,
|
| 520 |
+
"adv_clip_min": null,
|
| 521 |
+
"agent_name": "acfql",
|
| 522 |
+
"alpha": 500,
|
| 523 |
+
"batch_size": 256,
|
| 524 |
+
"best_of_n": 16,
|
| 525 |
+
"bon_agg": "mean",
|
| 526 |
+
"bon_sft": false,
|
| 527 |
+
"bon_sft_coeff": 1.0,
|
| 528 |
+
"bootstrap_ratio": 0.5,
|
| 529 |
+
"bptt_num_samples": 4,
|
| 530 |
+
"calql_q_subsample": 2,
|
| 531 |
+
"clip_grad_norm": 1000.0,
|
| 532 |
+
"clip_intermediate_actions": true,
|
| 533 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 534 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 535 |
+
"constant_noise_std": 0.01,
|
| 536 |
+
"cql_alpha": 5.0,
|
| 537 |
+
"cql_n_actions": 10,
|
| 538 |
+
"cql_temp": 1.0,
|
| 539 |
+
"critic_backbone": "mlp",
|
| 540 |
+
"critic_decay_steps": 5000,
|
| 541 |
+
"critic_end_value": 1e-08,
|
| 542 |
+
"critic_loss_type": "mse",
|
| 543 |
+
"critic_lr": 0.0003,
|
| 544 |
+
"critic_lr_partition": 0.0003,
|
| 545 |
+
"critic_obs": "state",
|
| 546 |
+
"critic_scheduler": "constant",
|
| 547 |
+
"critic_warmup_steps": 500,
|
| 548 |
+
"critic_weight_decay": 1e-05,
|
| 549 |
+
"denoised_clip_value": 1.0,
|
| 550 |
+
"discount": 0.999,
|
| 551 |
+
"edit_action_scale": 0.5,
|
| 552 |
+
"edit_actor_hidden_dims": null,
|
| 553 |
+
"edit_hidden_dims": [
|
| 554 |
+
256,
|
| 555 |
+
256,
|
| 556 |
+
256
|
| 557 |
+
],
|
| 558 |
+
"edit_init_temperature": 1.0,
|
| 559 |
+
"edit_layer_norm": true,
|
| 560 |
+
"edit_lr": 0.0003,
|
| 561 |
+
"edit_policy_bound": 0.1,
|
| 562 |
+
"edit_start_step": 100000,
|
| 563 |
+
"edit_target_entropy": null,
|
| 564 |
+
"edit_weight_decay": 0.0,
|
| 565 |
+
"encoder": null,
|
| 566 |
+
"entropy_backup": false,
|
| 567 |
+
"flow_steps": 10,
|
| 568 |
+
"fql_distillation_coeff": 10.0,
|
| 569 |
+
"fql_train_main_policy": true,
|
| 570 |
+
"freeze_vision_encoder": false,
|
| 571 |
+
"ft_flow_steps": 10,
|
| 572 |
+
"horizon_length": 8,
|
| 573 |
+
"init_temperature": 1.0,
|
| 574 |
+
"layer_norm": true,
|
| 575 |
+
"lr": 0.0003,
|
| 576 |
+
"lr_action_critic": null,
|
| 577 |
+
"lr_actor": null,
|
| 578 |
+
"lr_actor_bc_flow": null,
|
| 579 |
+
"lr_critic": null,
|
| 580 |
+
"lr_edit_actor": null,
|
| 581 |
+
"lr_noise_critic": null,
|
| 582 |
+
"max_noise_std": 0.01,
|
| 583 |
+
"mc_regression": false,
|
| 584 |
+
"mc_regression_coeff": 1.0,
|
| 585 |
+
"min_noise_std": 0.01,
|
| 586 |
+
"mip_q_ensemble": false,
|
| 587 |
+
"mip_q_noise_scale": 1.0,
|
| 588 |
+
"mip_q_t_star": 0.9,
|
| 589 |
+
"mip_t_star": 0.999,
|
| 590 |
+
"muon_beta": 0.95,
|
| 591 |
+
"muon_nesterov": true,
|
| 592 |
+
"muon_ns_steps": 5,
|
| 593 |
+
"n_action_samples": 8,
|
| 594 |
+
"n_edit_samples": 8,
|
| 595 |
+
"noise_critic_grad_steps": 1,
|
| 596 |
+
"noise_dim": null,
|
| 597 |
+
"noise_net_hidden_dims": [
|
| 598 |
+
256,
|
| 599 |
+
256
|
| 600 |
+
],
|
| 601 |
+
"noise_net_layer_norm": true,
|
| 602 |
+
"normalize_act_space_dimension": false,
|
| 603 |
+
"normalize_denoising_horizon": false,
|
| 604 |
+
"normalize_q_loss": false,
|
| 605 |
+
"num_bins": 256,
|
| 606 |
+
"num_ensemble_members": 10,
|
| 607 |
+
"num_qs": 10,
|
| 608 |
+
"ob_dims": [
|
| 609 |
+
137
|
| 610 |
+
],
|
| 611 |
+
"one_step_decay_steps": null,
|
| 612 |
+
"one_step_end_value": null,
|
| 613 |
+
"one_step_hidden_dims": [
|
| 614 |
+
512,
|
| 615 |
+
512,
|
| 616 |
+
512,
|
| 617 |
+
512
|
| 618 |
+
],
|
| 619 |
+
"one_step_layer_norm": false,
|
| 620 |
+
"one_step_lr": null,
|
| 621 |
+
"one_step_scheduler": null,
|
| 622 |
+
"one_step_warmup_steps": null,
|
| 623 |
+
"optimizer_action_critic": null,
|
| 624 |
+
"optimizer_actor": "adam",
|
| 625 |
+
"optimizer_actor_bc_flow": "adam",
|
| 626 |
+
"optimizer_critic": "adam",
|
| 627 |
+
"optimizer_edit_actor": null,
|
| 628 |
+
"paligemma_tokenizer_path": null,
|
| 629 |
+
"partition_optimizer": true,
|
| 630 |
+
"policy_type": "flow",
|
| 631 |
+
"ppo_batch_size": 256,
|
| 632 |
+
"ppo_lr": 4.5e-05,
|
| 633 |
+
"proprio_dim": null,
|
| 634 |
+
"q_agg": "mean",
|
| 635 |
+
"q_max": 1.0,
|
| 636 |
+
"q_min": 0.0,
|
| 637 |
+
"q_target_agg": "subset_min",
|
| 638 |
+
"q_target_replace": false,
|
| 639 |
+
"q_target_subset": 2,
|
| 640 |
+
"q_variance_reduction": false,
|
| 641 |
+
"q_vr_num_samples": 8,
|
| 642 |
+
"q_vr_reduction": "mean",
|
| 643 |
+
"randn_clip_value": 3.0,
|
| 644 |
+
"shortcut_denoising_steps": 4,
|
| 645 |
+
"shortcut_inference_steps": 4,
|
| 646 |
+
"simba_actor_hidden_dims": [
|
| 647 |
+
1024,
|
| 648 |
+
1024
|
| 649 |
+
],
|
| 650 |
+
"simba_critic_hidden_dims": [
|
| 651 |
+
1024,
|
| 652 |
+
1024,
|
| 653 |
+
1024
|
| 654 |
+
],
|
| 655 |
+
"subsample_bon": false,
|
| 656 |
+
"target_entropy": null,
|
| 657 |
+
"tau": 0.005,
|
| 658 |
+
"tf_pi_conditioning": "adaln",
|
| 659 |
+
"tf_pi_dropout": 0.0,
|
| 660 |
+
"tf_pi_embed_dim": 256,
|
| 661 |
+
"tf_pi_heads": 4,
|
| 662 |
+
"tf_pi_layers": 8,
|
| 663 |
+
"tf_q_conditioning": "adaln",
|
| 664 |
+
"tf_q_dropout": 0.0,
|
| 665 |
+
"tf_q_embed_dim": 256,
|
| 666 |
+
"tf_q_heads": 4,
|
| 667 |
+
"tf_q_layers": 8,
|
| 668 |
+
"time_embedding": "sinusoidal",
|
| 669 |
+
"time_embedding_dim": 32,
|
| 670 |
+
"train_one_step_in_offline": true,
|
| 671 |
+
"train_one_step_in_online": true,
|
| 672 |
+
"useFiLM": false,
|
| 673 |
+
"useSimBa": false,
|
| 674 |
+
"useTF": false,
|
| 675 |
+
"use_constant_noise": true,
|
| 676 |
+
"use_constant_scheduler_for_bc": true,
|
| 677 |
+
"use_denoiser": false,
|
| 678 |
+
"use_edit_actor": false,
|
| 679 |
+
"use_mip_q": false,
|
| 680 |
+
"use_muon": false,
|
| 681 |
+
"use_one_step_policy": false,
|
| 682 |
+
"use_shortcut": false,
|
| 683 |
+
"use_state": "proprio",
|
| 684 |
+
"use_success_buffer_q": false,
|
| 685 |
+
"value_hidden_dims": [
|
| 686 |
+
512,
|
| 687 |
+
512,
|
| 688 |
+
512
|
| 689 |
+
],
|
| 690 |
+
"vit_encoder": "minvit",
|
| 691 |
+
"vit_img_proj_dim": 0,
|
| 692 |
+
"vit_state_proj_dim": 0
|
| 693 |
+
},
|
| 694 |
+
"convention": "unified_env_step",
|
| 695 |
+
"eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
|
| 696 |
+
"gamma_env": 0.999,
|
| 697 |
+
"gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
|
| 698 |
+
"learner": "qc",
|
| 699 |
+
"optimizers": {
|
| 700 |
+
"actor_lr": 0.0001,
|
| 701 |
+
"critic_lr": 0.0003,
|
| 702 |
+
"reset_optimizers_with_lr": "not used",
|
| 703 |
+
"schedule": {
|
| 704 |
+
"actor": 0.0001,
|
| 705 |
+
"critic": 0.0003,
|
| 706 |
+
"decay_end_env_steps": 1606196,
|
| 707 |
+
"min_ratio": 0.1,
|
| 708 |
+
"schedule": "warmup_cosine",
|
| 709 |
+
"temperature": 0.0003,
|
| 710 |
+
"warmup_env_steps": 20000
|
| 711 |
+
}
|
| 712 |
+
},
|
| 713 |
+
"qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
|
| 714 |
+
"recipe": {
|
| 715 |
+
"derived": {
|
| 716 |
+
"agent.actor_num_samples": {
|
| 717 |
+
"after": 8,
|
| 718 |
+
"before": 32
|
| 719 |
+
},
|
| 720 |
+
"agent.actor_type": {
|
| 721 |
+
"after": "best-of-n",
|
| 722 |
+
"before": "upstream_yaml"
|
| 723 |
+
},
|
| 724 |
+
"agent.bon_sft": {
|
| 725 |
+
"after": false,
|
| 726 |
+
"before": "upstream_yaml"
|
| 727 |
+
}
|
| 728 |
+
},
|
| 729 |
+
"explicit_overrides": {},
|
| 730 |
+
"source": "third_party/ogpo/scripts/qc/toolhang.sh",
|
| 731 |
+
"upstream_exact": false
|
| 732 |
+
},
|
| 733 |
+
"reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
|
| 734 |
+
"rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
|
| 735 |
+
"scan_block": 100,
|
| 736 |
+
"start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
|
| 737 |
+
"success_terms": {
|
| 738 |
+
"bc_on_success_batch": false,
|
| 739 |
+
"critic_success_td": false,
|
| 740 |
+
"missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
|
| 741 |
+
},
|
| 742 |
+
"td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
|
| 743 |
+
"unified_notes": {
|
| 744 |
+
"critic_target_every_critic_step": {
|
| 745 |
+
"note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
|
| 746 |
+
"value": true
|
| 747 |
+
},
|
| 748 |
+
"success_td_polyak": {
|
| 749 |
+
"note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
|
| 750 |
+
"value": true
|
| 751 |
+
}
|
| 752 |
+
},
|
| 753 |
+
"upstream_config": {
|
| 754 |
+
"agent": {
|
| 755 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 756 |
+
"act_max": 3.0058202743530273,
|
| 757 |
+
"act_min": -2.9472272396087646,
|
| 758 |
+
"action_chunking": true,
|
| 759 |
+
"action_critic_hidden_dims": null,
|
| 760 |
+
"action_dim": null,
|
| 761 |
+
"action_num_qs": 10,
|
| 762 |
+
"action_repeat": false,
|
| 763 |
+
"actor_backbone": "mlp",
|
| 764 |
+
"actor_decay_steps": 50000,
|
| 765 |
+
"actor_end_value": 2e-05,
|
| 766 |
+
"actor_hidden_dims": [
|
| 767 |
+
512,
|
| 768 |
+
512,
|
| 769 |
+
512,
|
| 770 |
+
512,
|
| 771 |
+
512
|
| 772 |
+
],
|
| 773 |
+
"actor_layer_norm": false,
|
| 774 |
+
"actor_lr": 0.0003,
|
| 775 |
+
"actor_lr_partition": 0.0001,
|
| 776 |
+
"actor_num_samples": 16,
|
| 777 |
+
"actor_obs": "state",
|
| 778 |
+
"actor_scheduler": "cosine",
|
| 779 |
+
"actor_tau": 0.005,
|
| 780 |
+
"actor_type": "best-of-n",
|
| 781 |
+
"actor_warmup_steps": 2000,
|
| 782 |
+
"actor_weight_decay": 0.0,
|
| 783 |
+
"adv_clip_min": null,
|
| 784 |
+
"agent_name": "acfql",
|
| 785 |
+
"alpha": 500,
|
| 786 |
+
"batch_size": 256,
|
| 787 |
+
"best_of_n": 16,
|
| 788 |
+
"bon_agg": "mean",
|
| 789 |
+
"bon_sft": false,
|
| 790 |
+
"bon_sft_coeff": 1.0,
|
| 791 |
+
"bootstrap_ratio": 0.5,
|
| 792 |
+
"bptt_num_samples": 4,
|
| 793 |
+
"calql_q_subsample": 2,
|
| 794 |
+
"clip_grad_norm": 1000.0,
|
| 795 |
+
"clip_intermediate_actions": true,
|
| 796 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 797 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 798 |
+
"constant_noise_std": 0.01,
|
| 799 |
+
"cql_alpha": 5.0,
|
| 800 |
+
"cql_n_actions": 10,
|
| 801 |
+
"cql_temp": 1.0,
|
| 802 |
+
"critic_backbone": "mlp",
|
| 803 |
+
"critic_decay_steps": 5000,
|
| 804 |
+
"critic_end_value": 1e-08,
|
| 805 |
+
"critic_loss_type": "mse",
|
| 806 |
+
"critic_lr": 0.0003,
|
| 807 |
+
"critic_lr_partition": 0.0003,
|
| 808 |
+
"critic_obs": "state",
|
| 809 |
+
"critic_scheduler": "constant",
|
| 810 |
+
"critic_warmup_steps": 500,
|
| 811 |
+
"critic_weight_decay": 1e-05,
|
| 812 |
+
"denoised_clip_value": 1.0,
|
| 813 |
+
"discount": 0.999,
|
| 814 |
+
"edit_action_scale": 0.5,
|
| 815 |
+
"edit_actor_hidden_dims": null,
|
| 816 |
+
"edit_hidden_dims": [
|
| 817 |
+
256,
|
| 818 |
+
256,
|
| 819 |
+
256
|
| 820 |
+
],
|
| 821 |
+
"edit_init_temperature": 1.0,
|
| 822 |
+
"edit_layer_norm": true,
|
| 823 |
+
"edit_lr": 0.0003,
|
| 824 |
+
"edit_policy_bound": 0.1,
|
| 825 |
+
"edit_start_step": 100000,
|
| 826 |
+
"edit_target_entropy": null,
|
| 827 |
+
"edit_weight_decay": 0.0,
|
| 828 |
+
"encoder": null,
|
| 829 |
+
"entropy_backup": false,
|
| 830 |
+
"flow_steps": 10,
|
| 831 |
+
"fql_distillation_coeff": 10.0,
|
| 832 |
+
"fql_train_main_policy": true,
|
| 833 |
+
"freeze_vision_encoder": false,
|
| 834 |
+
"ft_flow_steps": 10,
|
| 835 |
+
"horizon_length": 8,
|
| 836 |
+
"init_temperature": 1.0,
|
| 837 |
+
"layer_norm": true,
|
| 838 |
+
"lr": 0.0003,
|
| 839 |
+
"lr_action_critic": null,
|
| 840 |
+
"lr_actor": null,
|
| 841 |
+
"lr_actor_bc_flow": null,
|
| 842 |
+
"lr_critic": null,
|
| 843 |
+
"lr_edit_actor": null,
|
| 844 |
+
"lr_noise_critic": null,
|
| 845 |
+
"max_noise_std": 0.01,
|
| 846 |
+
"mc_regression": false,
|
| 847 |
+
"mc_regression_coeff": 1.0,
|
| 848 |
+
"min_noise_std": 0.01,
|
| 849 |
+
"mip_q_ensemble": false,
|
| 850 |
+
"mip_q_noise_scale": 1.0,
|
| 851 |
+
"mip_q_t_star": 0.9,
|
| 852 |
+
"mip_t_star": 0.999,
|
| 853 |
+
"muon_beta": 0.95,
|
| 854 |
+
"muon_nesterov": true,
|
| 855 |
+
"muon_ns_steps": 5,
|
| 856 |
+
"n_action_samples": 8,
|
| 857 |
+
"n_edit_samples": 8,
|
| 858 |
+
"noise_critic_grad_steps": 1,
|
| 859 |
+
"noise_dim": null,
|
| 860 |
+
"noise_net_hidden_dims": [
|
| 861 |
+
256,
|
| 862 |
+
256
|
| 863 |
+
],
|
| 864 |
+
"noise_net_layer_norm": true,
|
| 865 |
+
"normalize_act_space_dimension": false,
|
| 866 |
+
"normalize_denoising_horizon": false,
|
| 867 |
+
"normalize_q_loss": false,
|
| 868 |
+
"num_bins": 256,
|
| 869 |
+
"num_ensemble_members": 10,
|
| 870 |
+
"num_qs": 10,
|
| 871 |
+
"ob_dims": null,
|
| 872 |
+
"one_step_decay_steps": null,
|
| 873 |
+
"one_step_end_value": null,
|
| 874 |
+
"one_step_hidden_dims": [
|
| 875 |
+
512,
|
| 876 |
+
512,
|
| 877 |
+
512,
|
| 878 |
+
512
|
| 879 |
+
],
|
| 880 |
+
"one_step_layer_norm": false,
|
| 881 |
+
"one_step_lr": null,
|
| 882 |
+
"one_step_scheduler": null,
|
| 883 |
+
"one_step_warmup_steps": null,
|
| 884 |
+
"optimizer_action_critic": null,
|
| 885 |
+
"optimizer_actor": "adam",
|
| 886 |
+
"optimizer_actor_bc_flow": "adam",
|
| 887 |
+
"optimizer_critic": "adam",
|
| 888 |
+
"optimizer_edit_actor": null,
|
| 889 |
+
"paligemma_tokenizer_path": null,
|
| 890 |
+
"partition_optimizer": true,
|
| 891 |
+
"policy_type": "flow",
|
| 892 |
+
"ppo_batch_size": 256,
|
| 893 |
+
"ppo_lr": 4.5e-05,
|
| 894 |
+
"proprio_dim": null,
|
| 895 |
+
"q_agg": "mean",
|
| 896 |
+
"q_max": null,
|
| 897 |
+
"q_min": null,
|
| 898 |
+
"q_target_agg": "subset_min",
|
| 899 |
+
"q_target_replace": false,
|
| 900 |
+
"q_target_subset": 2,
|
| 901 |
+
"q_variance_reduction": false,
|
| 902 |
+
"q_vr_num_samples": 8,
|
| 903 |
+
"q_vr_reduction": "mean",
|
| 904 |
+
"randn_clip_value": 3.0,
|
| 905 |
+
"shortcut_denoising_steps": 4,
|
| 906 |
+
"shortcut_inference_steps": 4,
|
| 907 |
+
"simba_actor_hidden_dims": [
|
| 908 |
+
1024,
|
| 909 |
+
1024
|
| 910 |
+
],
|
| 911 |
+
"simba_critic_hidden_dims": [
|
| 912 |
+
1024,
|
| 913 |
+
1024,
|
| 914 |
+
1024
|
| 915 |
+
],
|
| 916 |
+
"subsample_bon": false,
|
| 917 |
+
"target_entropy": null,
|
| 918 |
+
"tau": 0.005,
|
| 919 |
+
"tf_pi_conditioning": "adaln",
|
| 920 |
+
"tf_pi_dropout": 0.0,
|
| 921 |
+
"tf_pi_embed_dim": 256,
|
| 922 |
+
"tf_pi_heads": 4,
|
| 923 |
+
"tf_pi_layers": 8,
|
| 924 |
+
"tf_q_conditioning": "adaln",
|
| 925 |
+
"tf_q_dropout": 0.0,
|
| 926 |
+
"tf_q_embed_dim": 256,
|
| 927 |
+
"tf_q_heads": 4,
|
| 928 |
+
"tf_q_layers": 8,
|
| 929 |
+
"time_embedding": "sinusoidal",
|
| 930 |
+
"time_embedding_dim": 32,
|
| 931 |
+
"train_one_step_in_offline": true,
|
| 932 |
+
"train_one_step_in_online": true,
|
| 933 |
+
"useFiLM": false,
|
| 934 |
+
"useSimBa": false,
|
| 935 |
+
"useTF": false,
|
| 936 |
+
"use_constant_noise": true,
|
| 937 |
+
"use_constant_scheduler_for_bc": true,
|
| 938 |
+
"use_denoiser": false,
|
| 939 |
+
"use_edit_actor": false,
|
| 940 |
+
"use_mip_q": false,
|
| 941 |
+
"use_muon": false,
|
| 942 |
+
"use_one_step_policy": false,
|
| 943 |
+
"use_shortcut": false,
|
| 944 |
+
"use_state": "proprio",
|
| 945 |
+
"use_success_buffer_q": false,
|
| 946 |
+
"value_hidden_dims": [
|
| 947 |
+
512,
|
| 948 |
+
512,
|
| 949 |
+
512
|
| 950 |
+
],
|
| 951 |
+
"vit_encoder": "minvit",
|
| 952 |
+
"vit_img_proj_dim": 0,
|
| 953 |
+
"vit_state_proj_dim": 0
|
| 954 |
+
},
|
| 955 |
+
"bc": {
|
| 956 |
+
"clip_bc": true,
|
| 957 |
+
"clip_bc_threshold": 0.45,
|
| 958 |
+
"clip_bc_wrt": "ODE"
|
| 959 |
+
},
|
| 960 |
+
"buffer": {
|
| 961 |
+
"offline_ratio": 0.0,
|
| 962 |
+
"size": 2000000,
|
| 963 |
+
"start_training": 10000,
|
| 964 |
+
"utd_online": 1,
|
| 965 |
+
"utd_pi": null,
|
| 966 |
+
"utd_q": null,
|
| 967 |
+
"utd_v": null,
|
| 968 |
+
"utd_warmup": 1
|
| 969 |
+
},
|
| 970 |
+
"checkpoint": {
|
| 971 |
+
"ep_resume": 0,
|
| 972 |
+
"restore_actor_path": null,
|
| 973 |
+
"restore_critic_path": null,
|
| 974 |
+
"restore_encoder_path": null
|
| 975 |
+
},
|
| 976 |
+
"dataset": {
|
| 977 |
+
"num_offline_trajs": -1,
|
| 978 |
+
"ogbench_dir": null,
|
| 979 |
+
"p_aug": 0.0,
|
| 980 |
+
"proportion": 1.0,
|
| 981 |
+
"replace_interval": 1000
|
| 982 |
+
},
|
| 983 |
+
"env": {
|
| 984 |
+
"name": "tool_hang-ph-low_dim",
|
| 985 |
+
"post_success_steps": 0,
|
| 986 |
+
"rew_fn": "sparse",
|
| 987 |
+
"sparse": false
|
| 988 |
+
},
|
| 989 |
+
"eval": {
|
| 990 |
+
"eval_episodes": 2,
|
| 991 |
+
"n_eval_envs": 32,
|
| 992 |
+
"plot_q_vs_mc": false,
|
| 993 |
+
"video_episodes": 0,
|
| 994 |
+
"video_frame_skip": 3
|
| 995 |
+
},
|
| 996 |
+
"experiment": {
|
| 997 |
+
"log": true,
|
| 998 |
+
"project": "QC",
|
| 999 |
+
"run_group": "toolhang",
|
| 1000 |
+
"run_name": "qc_toolhang",
|
| 1001 |
+
"save_dir": "exp/ogpo",
|
| 1002 |
+
"seed": 1,
|
| 1003 |
+
"wandb_name": "qc_toolhang"
|
| 1004 |
+
},
|
| 1005 |
+
"intervals": {
|
| 1006 |
+
"eval_interval": 20000,
|
| 1007 |
+
"eval_interval_bc": 5000,
|
| 1008 |
+
"log_interval": 5000,
|
| 1009 |
+
"preemption_backup_interval": -1,
|
| 1010 |
+
"save_interval": 500000,
|
| 1011 |
+
"save_interval_bc": -1,
|
| 1012 |
+
"step_data_log_interval": -1
|
| 1013 |
+
},
|
| 1014 |
+
"success_buffer": {
|
| 1015 |
+
"batch_size": 256,
|
| 1016 |
+
"use_success_buffer": false
|
| 1017 |
+
},
|
| 1018 |
+
"training": {
|
| 1019 |
+
"bc_only": false,
|
| 1020 |
+
"bc_pi_steps": 0,
|
| 1021 |
+
"bc_q_steps": 0,
|
| 1022 |
+
"bc_refine_steps": 0,
|
| 1023 |
+
"calql_steps": 0,
|
| 1024 |
+
"force_jax_sync": true,
|
| 1025 |
+
"gpu_mode": "single",
|
| 1026 |
+
"online_steps": 5000000,
|
| 1027 |
+
"q_warmup_steps": 0
|
| 1028 |
+
}
|
| 1029 |
+
},
|
| 1030 |
+
"upstream_source": {
|
| 1031 |
+
"ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
|
| 1032 |
+
"patches_sha256": {
|
| 1033 |
+
"ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
|
| 1034 |
+
"ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
|
| 1035 |
+
"ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
|
| 1036 |
+
"ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
|
| 1037 |
+
"ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
|
| 1038 |
+
}
|
| 1039 |
+
}
|
| 1040 |
+
},
|
| 1041 |
+
"observation_mode": "pixel_state",
|
| 1042 |
+
"offline_buffer": {
|
| 1043 |
+
"actions_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1044 |
+
"aux_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1045 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 1046 |
+
"chunks": 6171,
|
| 1047 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1048 |
+
"decision_ref": "[1017-S2OFFBC]",
|
| 1049 |
+
"env_steps": 48984,
|
| 1050 |
+
"features_sha256": "fb059cee8121cd0039a4ac218f1a19de767b07cf81010471e4342480fc9119fa",
|
| 1051 |
+
"horizon": 8,
|
| 1052 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 1053 |
+
"learner_coordinates": "JaxStudy2Learner:qc",
|
| 1054 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1055 |
+
"ratio": {
|
| 1056 |
+
"adaptive_ratio": false,
|
| 1057 |
+
"rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
|
| 1058 |
+
},
|
| 1059 |
+
"selection": {
|
| 1060 |
+
"cohort_size": 100,
|
| 1061 |
+
"demos": [
|
| 1062 |
+
"demo_0",
|
| 1063 |
+
"demo_2",
|
| 1064 |
+
"demo_5",
|
| 1065 |
+
"demo_7",
|
| 1066 |
+
"demo_9",
|
| 1067 |
+
"demo_10",
|
| 1068 |
+
"demo_13",
|
| 1069 |
+
"demo_16",
|
| 1070 |
+
"demo_17",
|
| 1071 |
+
"demo_18",
|
| 1072 |
+
"demo_20",
|
| 1073 |
+
"demo_23",
|
| 1074 |
+
"demo_27",
|
| 1075 |
+
"demo_28",
|
| 1076 |
+
"demo_29",
|
| 1077 |
+
"demo_30",
|
| 1078 |
+
"demo_31",
|
| 1079 |
+
"demo_32",
|
| 1080 |
+
"demo_34",
|
| 1081 |
+
"demo_35",
|
| 1082 |
+
"demo_37",
|
| 1083 |
+
"demo_40",
|
| 1084 |
+
"demo_41",
|
| 1085 |
+
"demo_43",
|
| 1086 |
+
"demo_46",
|
| 1087 |
+
"demo_47",
|
| 1088 |
+
"demo_50",
|
| 1089 |
+
"demo_51",
|
| 1090 |
+
"demo_56",
|
| 1091 |
+
"demo_57",
|
| 1092 |
+
"demo_58",
|
| 1093 |
+
"demo_67",
|
| 1094 |
+
"demo_71",
|
| 1095 |
+
"demo_74",
|
| 1096 |
+
"demo_76",
|
| 1097 |
+
"demo_78",
|
| 1098 |
+
"demo_80",
|
| 1099 |
+
"demo_82",
|
| 1100 |
+
"demo_83",
|
| 1101 |
+
"demo_85",
|
| 1102 |
+
"demo_86",
|
| 1103 |
+
"demo_87",
|
| 1104 |
+
"demo_93",
|
| 1105 |
+
"demo_97",
|
| 1106 |
+
"demo_98",
|
| 1107 |
+
"demo_101",
|
| 1108 |
+
"demo_102",
|
| 1109 |
+
"demo_107",
|
| 1110 |
+
"demo_110",
|
| 1111 |
+
"demo_111",
|
| 1112 |
+
"demo_112",
|
| 1113 |
+
"demo_113",
|
| 1114 |
+
"demo_115",
|
| 1115 |
+
"demo_116",
|
| 1116 |
+
"demo_119",
|
| 1117 |
+
"demo_122",
|
| 1118 |
+
"demo_123",
|
| 1119 |
+
"demo_126",
|
| 1120 |
+
"demo_127",
|
| 1121 |
+
"demo_128",
|
| 1122 |
+
"demo_130",
|
| 1123 |
+
"demo_131",
|
| 1124 |
+
"demo_132",
|
| 1125 |
+
"demo_133",
|
| 1126 |
+
"demo_134",
|
| 1127 |
+
"demo_135",
|
| 1128 |
+
"demo_138",
|
| 1129 |
+
"demo_140",
|
| 1130 |
+
"demo_141",
|
| 1131 |
+
"demo_145",
|
| 1132 |
+
"demo_147",
|
| 1133 |
+
"demo_149",
|
| 1134 |
+
"demo_151",
|
| 1135 |
+
"demo_152",
|
| 1136 |
+
"demo_154",
|
| 1137 |
+
"demo_156",
|
| 1138 |
+
"demo_157",
|
| 1139 |
+
"demo_159",
|
| 1140 |
+
"demo_161",
|
| 1141 |
+
"demo_163",
|
| 1142 |
+
"demo_164",
|
| 1143 |
+
"demo_171",
|
| 1144 |
+
"demo_172",
|
| 1145 |
+
"demo_173",
|
| 1146 |
+
"demo_174",
|
| 1147 |
+
"demo_175",
|
| 1148 |
+
"demo_176",
|
| 1149 |
+
"demo_177",
|
| 1150 |
+
"demo_179",
|
| 1151 |
+
"demo_181",
|
| 1152 |
+
"demo_182",
|
| 1153 |
+
"demo_183",
|
| 1154 |
+
"demo_186",
|
| 1155 |
+
"demo_188",
|
| 1156 |
+
"demo_190",
|
| 1157 |
+
"demo_191",
|
| 1158 |
+
"demo_192",
|
| 1159 |
+
"demo_193",
|
| 1160 |
+
"demo_196",
|
| 1161 |
+
"demo_199"
|
| 1162 |
+
],
|
| 1163 |
+
"manifest": "/ssd/WorkSpace/RL-Postrain/checkpoints/huggingface/yangfy0627/RL-postTrain/e62671ed1cf646b453660b698c22ca4d273b5ccc/Robomimic/study1/ToolHang/diverse100/seed_1/data_manifest.json",
|
| 1164 |
+
"manifest_sha256": "9233d5824205e29f1afa721d8573fe19805a1a8e7829c296af1dd0b74a1d29b4",
|
| 1165 |
+
"regime": "diverse100",
|
| 1166 |
+
"source": "bc_regime",
|
| 1167 |
+
"source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1168 |
+
"split": {
|
| 1169 |
+
"training": 90,
|
| 1170 |
+
"validation": 10
|
| 1171 |
+
},
|
| 1172 |
+
"task": "tool_hang"
|
| 1173 |
+
}
|
| 1174 |
+
},
|
| 1175 |
+
"offline_role": "mixed",
|
| 1176 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1177 |
+
"smoke": false,
|
| 1178 |
+
"task": "tool_hang",
|
| 1179 |
+
"task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
|
| 1180 |
+
},
|
| 1181 |
+
"seed": 1,
|
| 1182 |
+
"start_policy": "base_cfm_ode",
|
| 1183 |
+
"variant": {
|
| 1184 |
+
"flags": {
|
| 1185 |
+
"bon_sft": false,
|
| 1186 |
+
"use_success_buffer": false
|
| 1187 |
+
},
|
| 1188 |
+
"learner": "qc",
|
| 1189 |
+
"name": "qc",
|
| 1190 |
+
"phase": 1,
|
| 1191 |
+
"spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
|
| 1192 |
+
"version": "study2_variants_v4"
|
| 1193 |
+
},
|
| 1194 |
+
"variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
|
| 1195 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/final/state.pkl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8f507af6fb47652d3b5b512d06c8bc51d2da9d2bab494670f88f9b9ce859104d
|
| 3 |
+
size 207222745
|
RL/ToolHang100/QC/diverse100/seed_1/frozen_base.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"after": "719a7185a0d6a4877c76a5e1d9b1adb163a3d863d28bced54e79d534e8db6873",
|
| 3 |
+
"before": "719a7185a0d6a4877c76a5e1d9b1adb163a3d863d28bced54e79d534e8db6873",
|
| 4 |
+
"checkpoint_unchanged": true,
|
| 5 |
+
"unchanged": true
|
| 6 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/offline_buffer.json
ADDED
|
@@ -0,0 +1,847 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"identity": {
|
| 3 |
+
"actions_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 4 |
+
"aux_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 5 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 6 |
+
"chunks": 6171,
|
| 7 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 8 |
+
"decision_ref": "[1017-S2OFFBC]",
|
| 9 |
+
"env_steps": 48984,
|
| 10 |
+
"features_sha256": "fb059cee8121cd0039a4ac218f1a19de767b07cf81010471e4342480fc9119fa",
|
| 11 |
+
"horizon": 8,
|
| 12 |
+
"learner_coordinates": "JaxStudy2Learner:qc",
|
| 13 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 14 |
+
"ratio": {
|
| 15 |
+
"adaptive_ratio": false,
|
| 16 |
+
"rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
|
| 17 |
+
},
|
| 18 |
+
"selection": {
|
| 19 |
+
"cohort_size": 100,
|
| 20 |
+
"demos": [
|
| 21 |
+
"demo_0",
|
| 22 |
+
"demo_2",
|
| 23 |
+
"demo_5",
|
| 24 |
+
"demo_7",
|
| 25 |
+
"demo_9",
|
| 26 |
+
"demo_10",
|
| 27 |
+
"demo_13",
|
| 28 |
+
"demo_16",
|
| 29 |
+
"demo_17",
|
| 30 |
+
"demo_18",
|
| 31 |
+
"demo_20",
|
| 32 |
+
"demo_23",
|
| 33 |
+
"demo_27",
|
| 34 |
+
"demo_28",
|
| 35 |
+
"demo_29",
|
| 36 |
+
"demo_30",
|
| 37 |
+
"demo_31",
|
| 38 |
+
"demo_32",
|
| 39 |
+
"demo_34",
|
| 40 |
+
"demo_35",
|
| 41 |
+
"demo_37",
|
| 42 |
+
"demo_40",
|
| 43 |
+
"demo_41",
|
| 44 |
+
"demo_43",
|
| 45 |
+
"demo_46",
|
| 46 |
+
"demo_47",
|
| 47 |
+
"demo_50",
|
| 48 |
+
"demo_51",
|
| 49 |
+
"demo_56",
|
| 50 |
+
"demo_57",
|
| 51 |
+
"demo_58",
|
| 52 |
+
"demo_67",
|
| 53 |
+
"demo_71",
|
| 54 |
+
"demo_74",
|
| 55 |
+
"demo_76",
|
| 56 |
+
"demo_78",
|
| 57 |
+
"demo_80",
|
| 58 |
+
"demo_82",
|
| 59 |
+
"demo_83",
|
| 60 |
+
"demo_85",
|
| 61 |
+
"demo_86",
|
| 62 |
+
"demo_87",
|
| 63 |
+
"demo_93",
|
| 64 |
+
"demo_97",
|
| 65 |
+
"demo_98",
|
| 66 |
+
"demo_101",
|
| 67 |
+
"demo_102",
|
| 68 |
+
"demo_107",
|
| 69 |
+
"demo_110",
|
| 70 |
+
"demo_111",
|
| 71 |
+
"demo_112",
|
| 72 |
+
"demo_113",
|
| 73 |
+
"demo_115",
|
| 74 |
+
"demo_116",
|
| 75 |
+
"demo_119",
|
| 76 |
+
"demo_122",
|
| 77 |
+
"demo_123",
|
| 78 |
+
"demo_126",
|
| 79 |
+
"demo_127",
|
| 80 |
+
"demo_128",
|
| 81 |
+
"demo_130",
|
| 82 |
+
"demo_131",
|
| 83 |
+
"demo_132",
|
| 84 |
+
"demo_133",
|
| 85 |
+
"demo_134",
|
| 86 |
+
"demo_135",
|
| 87 |
+
"demo_138",
|
| 88 |
+
"demo_140",
|
| 89 |
+
"demo_141",
|
| 90 |
+
"demo_145",
|
| 91 |
+
"demo_147",
|
| 92 |
+
"demo_149",
|
| 93 |
+
"demo_151",
|
| 94 |
+
"demo_152",
|
| 95 |
+
"demo_154",
|
| 96 |
+
"demo_156",
|
| 97 |
+
"demo_157",
|
| 98 |
+
"demo_159",
|
| 99 |
+
"demo_161",
|
| 100 |
+
"demo_163",
|
| 101 |
+
"demo_164",
|
| 102 |
+
"demo_171",
|
| 103 |
+
"demo_172",
|
| 104 |
+
"demo_173",
|
| 105 |
+
"demo_174",
|
| 106 |
+
"demo_175",
|
| 107 |
+
"demo_176",
|
| 108 |
+
"demo_177",
|
| 109 |
+
"demo_179",
|
| 110 |
+
"demo_181",
|
| 111 |
+
"demo_182",
|
| 112 |
+
"demo_183",
|
| 113 |
+
"demo_186",
|
| 114 |
+
"demo_188",
|
| 115 |
+
"demo_190",
|
| 116 |
+
"demo_191",
|
| 117 |
+
"demo_192",
|
| 118 |
+
"demo_193",
|
| 119 |
+
"demo_196",
|
| 120 |
+
"demo_199"
|
| 121 |
+
],
|
| 122 |
+
"manifest": "/ssd/WorkSpace/RL-Postrain/checkpoints/huggingface/yangfy0627/RL-postTrain/e62671ed1cf646b453660b698c22ca4d273b5ccc/Robomimic/study1/ToolHang/diverse100/seed_1/data_manifest.json",
|
| 123 |
+
"manifest_sha256": "9233d5824205e29f1afa721d8573fe19805a1a8e7829c296af1dd0b74a1d29b4",
|
| 124 |
+
"regime": "diverse100",
|
| 125 |
+
"source": "bc_regime",
|
| 126 |
+
"source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 127 |
+
"split": {
|
| 128 |
+
"training": 90,
|
| 129 |
+
"validation": 10
|
| 130 |
+
},
|
| 131 |
+
"task": "tool_hang"
|
| 132 |
+
}
|
| 133 |
+
},
|
| 134 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 135 |
+
"stats": {
|
| 136 |
+
"chunks": 6171,
|
| 137 |
+
"dataset": "/ssd/WorkSpace/RL-Postrain/data/robomimic/tool_hang/ph/low_dim_v15.hdf5",
|
| 138 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 139 |
+
"env_steps": 48984,
|
| 140 |
+
"per_demo": [
|
| 141 |
+
{
|
| 142 |
+
"chunks": 85,
|
| 143 |
+
"demo": "demo_0",
|
| 144 |
+
"env_steps": 677,
|
| 145 |
+
"frames": 681,
|
| 146 |
+
"terminal_next_frame": 677
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"chunks": 92,
|
| 150 |
+
"demo": "demo_2",
|
| 151 |
+
"env_steps": 733,
|
| 152 |
+
"frames": 737,
|
| 153 |
+
"terminal_next_frame": 733
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"chunks": 85,
|
| 157 |
+
"demo": "demo_5",
|
| 158 |
+
"env_steps": 677,
|
| 159 |
+
"frames": 681,
|
| 160 |
+
"terminal_next_frame": 677
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"chunks": 78,
|
| 164 |
+
"demo": "demo_7",
|
| 165 |
+
"env_steps": 618,
|
| 166 |
+
"frames": 622,
|
| 167 |
+
"terminal_next_frame": 618
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"chunks": 78,
|
| 171 |
+
"demo": "demo_9",
|
| 172 |
+
"env_steps": 619,
|
| 173 |
+
"frames": 623,
|
| 174 |
+
"terminal_next_frame": 619
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"chunks": 62,
|
| 178 |
+
"demo": "demo_10",
|
| 179 |
+
"env_steps": 493,
|
| 180 |
+
"frames": 497,
|
| 181 |
+
"terminal_next_frame": 493
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"chunks": 64,
|
| 185 |
+
"demo": "demo_13",
|
| 186 |
+
"env_steps": 510,
|
| 187 |
+
"frames": 514,
|
| 188 |
+
"terminal_next_frame": 510
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"chunks": 84,
|
| 192 |
+
"demo": "demo_16",
|
| 193 |
+
"env_steps": 665,
|
| 194 |
+
"frames": 676,
|
| 195 |
+
"terminal_next_frame": 665
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"chunks": 84,
|
| 199 |
+
"demo": "demo_17",
|
| 200 |
+
"env_steps": 668,
|
| 201 |
+
"frames": 677,
|
| 202 |
+
"terminal_next_frame": 668
|
| 203 |
+
},
|
| 204 |
+
{
|
| 205 |
+
"chunks": 76,
|
| 206 |
+
"demo": "demo_18",
|
| 207 |
+
"env_steps": 605,
|
| 208 |
+
"frames": 632,
|
| 209 |
+
"terminal_next_frame": 605
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"chunks": 71,
|
| 213 |
+
"demo": "demo_20",
|
| 214 |
+
"env_steps": 561,
|
| 215 |
+
"frames": 565,
|
| 216 |
+
"terminal_next_frame": 561
|
| 217 |
+
},
|
| 218 |
+
{
|
| 219 |
+
"chunks": 93,
|
| 220 |
+
"demo": "demo_23",
|
| 221 |
+
"env_steps": 738,
|
| 222 |
+
"frames": 742,
|
| 223 |
+
"terminal_next_frame": 738
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"chunks": 59,
|
| 227 |
+
"demo": "demo_27",
|
| 228 |
+
"env_steps": 465,
|
| 229 |
+
"frames": 469,
|
| 230 |
+
"terminal_next_frame": 465
|
| 231 |
+
},
|
| 232 |
+
{
|
| 233 |
+
"chunks": 64,
|
| 234 |
+
"demo": "demo_28",
|
| 235 |
+
"env_steps": 505,
|
| 236 |
+
"frames": 512,
|
| 237 |
+
"terminal_next_frame": 505
|
| 238 |
+
},
|
| 239 |
+
{
|
| 240 |
+
"chunks": 66,
|
| 241 |
+
"demo": "demo_29",
|
| 242 |
+
"env_steps": 526,
|
| 243 |
+
"frames": 532,
|
| 244 |
+
"terminal_next_frame": 526
|
| 245 |
+
},
|
| 246 |
+
{
|
| 247 |
+
"chunks": 51,
|
| 248 |
+
"demo": "demo_30",
|
| 249 |
+
"env_steps": 405,
|
| 250 |
+
"frames": 409,
|
| 251 |
+
"terminal_next_frame": 405
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"chunks": 58,
|
| 255 |
+
"demo": "demo_31",
|
| 256 |
+
"env_steps": 458,
|
| 257 |
+
"frames": 476,
|
| 258 |
+
"terminal_next_frame": 458
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"chunks": 59,
|
| 262 |
+
"demo": "demo_32",
|
| 263 |
+
"env_steps": 467,
|
| 264 |
+
"frames": 473,
|
| 265 |
+
"terminal_next_frame": 467
|
| 266 |
+
},
|
| 267 |
+
{
|
| 268 |
+
"chunks": 72,
|
| 269 |
+
"demo": "demo_34",
|
| 270 |
+
"env_steps": 570,
|
| 271 |
+
"frames": 574,
|
| 272 |
+
"terminal_next_frame": 570
|
| 273 |
+
},
|
| 274 |
+
{
|
| 275 |
+
"chunks": 74,
|
| 276 |
+
"demo": "demo_35",
|
| 277 |
+
"env_steps": 585,
|
| 278 |
+
"frames": 589,
|
| 279 |
+
"terminal_next_frame": 585
|
| 280 |
+
},
|
| 281 |
+
{
|
| 282 |
+
"chunks": 71,
|
| 283 |
+
"demo": "demo_37",
|
| 284 |
+
"env_steps": 563,
|
| 285 |
+
"frames": 567,
|
| 286 |
+
"terminal_next_frame": 563
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"chunks": 81,
|
| 290 |
+
"demo": "demo_40",
|
| 291 |
+
"env_steps": 645,
|
| 292 |
+
"frames": 649,
|
| 293 |
+
"terminal_next_frame": 645
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"chunks": 68,
|
| 297 |
+
"demo": "demo_41",
|
| 298 |
+
"env_steps": 538,
|
| 299 |
+
"frames": 538,
|
| 300 |
+
"terminal_next_frame": 537
|
| 301 |
+
},
|
| 302 |
+
{
|
| 303 |
+
"chunks": 62,
|
| 304 |
+
"demo": "demo_43",
|
| 305 |
+
"env_steps": 491,
|
| 306 |
+
"frames": 495,
|
| 307 |
+
"terminal_next_frame": 491
|
| 308 |
+
},
|
| 309 |
+
{
|
| 310 |
+
"chunks": 56,
|
| 311 |
+
"demo": "demo_46",
|
| 312 |
+
"env_steps": 446,
|
| 313 |
+
"frames": 450,
|
| 314 |
+
"terminal_next_frame": 446
|
| 315 |
+
},
|
| 316 |
+
{
|
| 317 |
+
"chunks": 67,
|
| 318 |
+
"demo": "demo_47",
|
| 319 |
+
"env_steps": 535,
|
| 320 |
+
"frames": 539,
|
| 321 |
+
"terminal_next_frame": 535
|
| 322 |
+
},
|
| 323 |
+
{
|
| 324 |
+
"chunks": 54,
|
| 325 |
+
"demo": "demo_50",
|
| 326 |
+
"env_steps": 432,
|
| 327 |
+
"frames": 509,
|
| 328 |
+
"terminal_next_frame": 432
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"chunks": 63,
|
| 332 |
+
"demo": "demo_51",
|
| 333 |
+
"env_steps": 498,
|
| 334 |
+
"frames": 502,
|
| 335 |
+
"terminal_next_frame": 498
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"chunks": 59,
|
| 339 |
+
"demo": "demo_56",
|
| 340 |
+
"env_steps": 465,
|
| 341 |
+
"frames": 469,
|
| 342 |
+
"terminal_next_frame": 465
|
| 343 |
+
},
|
| 344 |
+
{
|
| 345 |
+
"chunks": 59,
|
| 346 |
+
"demo": "demo_57",
|
| 347 |
+
"env_steps": 472,
|
| 348 |
+
"frames": 476,
|
| 349 |
+
"terminal_next_frame": 472
|
| 350 |
+
},
|
| 351 |
+
{
|
| 352 |
+
"chunks": 54,
|
| 353 |
+
"demo": "demo_58",
|
| 354 |
+
"env_steps": 429,
|
| 355 |
+
"frames": 433,
|
| 356 |
+
"terminal_next_frame": 429
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"chunks": 59,
|
| 360 |
+
"demo": "demo_67",
|
| 361 |
+
"env_steps": 470,
|
| 362 |
+
"frames": 474,
|
| 363 |
+
"terminal_next_frame": 470
|
| 364 |
+
},
|
| 365 |
+
{
|
| 366 |
+
"chunks": 70,
|
| 367 |
+
"demo": "demo_71",
|
| 368 |
+
"env_steps": 555,
|
| 369 |
+
"frames": 562,
|
| 370 |
+
"terminal_next_frame": 555
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"chunks": 59,
|
| 374 |
+
"demo": "demo_74",
|
| 375 |
+
"env_steps": 468,
|
| 376 |
+
"frames": 470,
|
| 377 |
+
"terminal_next_frame": 468
|
| 378 |
+
},
|
| 379 |
+
{
|
| 380 |
+
"chunks": 52,
|
| 381 |
+
"demo": "demo_76",
|
| 382 |
+
"env_steps": 412,
|
| 383 |
+
"frames": 439,
|
| 384 |
+
"terminal_next_frame": 412
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"chunks": 56,
|
| 388 |
+
"demo": "demo_78",
|
| 389 |
+
"env_steps": 448,
|
| 390 |
+
"frames": 452,
|
| 391 |
+
"terminal_next_frame": 448
|
| 392 |
+
},
|
| 393 |
+
{
|
| 394 |
+
"chunks": 67,
|
| 395 |
+
"demo": "demo_80",
|
| 396 |
+
"env_steps": 533,
|
| 397 |
+
"frames": 537,
|
| 398 |
+
"terminal_next_frame": 533
|
| 399 |
+
},
|
| 400 |
+
{
|
| 401 |
+
"chunks": 52,
|
| 402 |
+
"demo": "demo_82",
|
| 403 |
+
"env_steps": 416,
|
| 404 |
+
"frames": 422,
|
| 405 |
+
"terminal_next_frame": 416
|
| 406 |
+
},
|
| 407 |
+
{
|
| 408 |
+
"chunks": 51,
|
| 409 |
+
"demo": "demo_83",
|
| 410 |
+
"env_steps": 408,
|
| 411 |
+
"frames": 440,
|
| 412 |
+
"terminal_next_frame": 408
|
| 413 |
+
},
|
| 414 |
+
{
|
| 415 |
+
"chunks": 61,
|
| 416 |
+
"demo": "demo_85",
|
| 417 |
+
"env_steps": 484,
|
| 418 |
+
"frames": 488,
|
| 419 |
+
"terminal_next_frame": 484
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"chunks": 60,
|
| 423 |
+
"demo": "demo_86",
|
| 424 |
+
"env_steps": 475,
|
| 425 |
+
"frames": 479,
|
| 426 |
+
"terminal_next_frame": 475
|
| 427 |
+
},
|
| 428 |
+
{
|
| 429 |
+
"chunks": 55,
|
| 430 |
+
"demo": "demo_87",
|
| 431 |
+
"env_steps": 438,
|
| 432 |
+
"frames": 443,
|
| 433 |
+
"terminal_next_frame": 438
|
| 434 |
+
},
|
| 435 |
+
{
|
| 436 |
+
"chunks": 47,
|
| 437 |
+
"demo": "demo_93",
|
| 438 |
+
"env_steps": 374,
|
| 439 |
+
"frames": 378,
|
| 440 |
+
"terminal_next_frame": 374
|
| 441 |
+
},
|
| 442 |
+
{
|
| 443 |
+
"chunks": 76,
|
| 444 |
+
"demo": "demo_97",
|
| 445 |
+
"env_steps": 601,
|
| 446 |
+
"frames": 608,
|
| 447 |
+
"terminal_next_frame": 601
|
| 448 |
+
},
|
| 449 |
+
{
|
| 450 |
+
"chunks": 49,
|
| 451 |
+
"demo": "demo_98",
|
| 452 |
+
"env_steps": 390,
|
| 453 |
+
"frames": 394,
|
| 454 |
+
"terminal_next_frame": 390
|
| 455 |
+
},
|
| 456 |
+
{
|
| 457 |
+
"chunks": 52,
|
| 458 |
+
"demo": "demo_101",
|
| 459 |
+
"env_steps": 410,
|
| 460 |
+
"frames": 414,
|
| 461 |
+
"terminal_next_frame": 410
|
| 462 |
+
},
|
| 463 |
+
{
|
| 464 |
+
"chunks": 74,
|
| 465 |
+
"demo": "demo_102",
|
| 466 |
+
"env_steps": 586,
|
| 467 |
+
"frames": 590,
|
| 468 |
+
"terminal_next_frame": 586
|
| 469 |
+
},
|
| 470 |
+
{
|
| 471 |
+
"chunks": 47,
|
| 472 |
+
"demo": "demo_107",
|
| 473 |
+
"env_steps": 371,
|
| 474 |
+
"frames": 375,
|
| 475 |
+
"terminal_next_frame": 371
|
| 476 |
+
},
|
| 477 |
+
{
|
| 478 |
+
"chunks": 47,
|
| 479 |
+
"demo": "demo_110",
|
| 480 |
+
"env_steps": 373,
|
| 481 |
+
"frames": 377,
|
| 482 |
+
"terminal_next_frame": 373
|
| 483 |
+
},
|
| 484 |
+
{
|
| 485 |
+
"chunks": 51,
|
| 486 |
+
"demo": "demo_111",
|
| 487 |
+
"env_steps": 401,
|
| 488 |
+
"frames": 405,
|
| 489 |
+
"terminal_next_frame": 401
|
| 490 |
+
},
|
| 491 |
+
{
|
| 492 |
+
"chunks": 61,
|
| 493 |
+
"demo": "demo_112",
|
| 494 |
+
"env_steps": 482,
|
| 495 |
+
"frames": 486,
|
| 496 |
+
"terminal_next_frame": 482
|
| 497 |
+
},
|
| 498 |
+
{
|
| 499 |
+
"chunks": 55,
|
| 500 |
+
"demo": "demo_113",
|
| 501 |
+
"env_steps": 436,
|
| 502 |
+
"frames": 440,
|
| 503 |
+
"terminal_next_frame": 436
|
| 504 |
+
},
|
| 505 |
+
{
|
| 506 |
+
"chunks": 59,
|
| 507 |
+
"demo": "demo_115",
|
| 508 |
+
"env_steps": 466,
|
| 509 |
+
"frames": 470,
|
| 510 |
+
"terminal_next_frame": 466
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"chunks": 55,
|
| 514 |
+
"demo": "demo_116",
|
| 515 |
+
"env_steps": 434,
|
| 516 |
+
"frames": 472,
|
| 517 |
+
"terminal_next_frame": 434
|
| 518 |
+
},
|
| 519 |
+
{
|
| 520 |
+
"chunks": 67,
|
| 521 |
+
"demo": "demo_119",
|
| 522 |
+
"env_steps": 531,
|
| 523 |
+
"frames": 535,
|
| 524 |
+
"terminal_next_frame": 531
|
| 525 |
+
},
|
| 526 |
+
{
|
| 527 |
+
"chunks": 45,
|
| 528 |
+
"demo": "demo_122",
|
| 529 |
+
"env_steps": 356,
|
| 530 |
+
"frames": 370,
|
| 531 |
+
"terminal_next_frame": 356
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"chunks": 48,
|
| 535 |
+
"demo": "demo_123",
|
| 536 |
+
"env_steps": 383,
|
| 537 |
+
"frames": 415,
|
| 538 |
+
"terminal_next_frame": 383
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"chunks": 46,
|
| 542 |
+
"demo": "demo_126",
|
| 543 |
+
"env_steps": 365,
|
| 544 |
+
"frames": 371,
|
| 545 |
+
"terminal_next_frame": 365
|
| 546 |
+
},
|
| 547 |
+
{
|
| 548 |
+
"chunks": 50,
|
| 549 |
+
"demo": "demo_127",
|
| 550 |
+
"env_steps": 396,
|
| 551 |
+
"frames": 421,
|
| 552 |
+
"terminal_next_frame": 396
|
| 553 |
+
},
|
| 554 |
+
{
|
| 555 |
+
"chunks": 71,
|
| 556 |
+
"demo": "demo_128",
|
| 557 |
+
"env_steps": 563,
|
| 558 |
+
"frames": 567,
|
| 559 |
+
"terminal_next_frame": 563
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"chunks": 60,
|
| 563 |
+
"demo": "demo_130",
|
| 564 |
+
"env_steps": 474,
|
| 565 |
+
"frames": 478,
|
| 566 |
+
"terminal_next_frame": 474
|
| 567 |
+
},
|
| 568 |
+
{
|
| 569 |
+
"chunks": 42,
|
| 570 |
+
"demo": "demo_131",
|
| 571 |
+
"env_steps": 332,
|
| 572 |
+
"frames": 336,
|
| 573 |
+
"terminal_next_frame": 332
|
| 574 |
+
},
|
| 575 |
+
{
|
| 576 |
+
"chunks": 54,
|
| 577 |
+
"demo": "demo_132",
|
| 578 |
+
"env_steps": 427,
|
| 579 |
+
"frames": 431,
|
| 580 |
+
"terminal_next_frame": 427
|
| 581 |
+
},
|
| 582 |
+
{
|
| 583 |
+
"chunks": 62,
|
| 584 |
+
"demo": "demo_133",
|
| 585 |
+
"env_steps": 494,
|
| 586 |
+
"frames": 498,
|
| 587 |
+
"terminal_next_frame": 494
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"chunks": 41,
|
| 591 |
+
"demo": "demo_134",
|
| 592 |
+
"env_steps": 328,
|
| 593 |
+
"frames": 332,
|
| 594 |
+
"terminal_next_frame": 328
|
| 595 |
+
},
|
| 596 |
+
{
|
| 597 |
+
"chunks": 52,
|
| 598 |
+
"demo": "demo_135",
|
| 599 |
+
"env_steps": 411,
|
| 600 |
+
"frames": 415,
|
| 601 |
+
"terminal_next_frame": 411
|
| 602 |
+
},
|
| 603 |
+
{
|
| 604 |
+
"chunks": 70,
|
| 605 |
+
"demo": "demo_138",
|
| 606 |
+
"env_steps": 554,
|
| 607 |
+
"frames": 580,
|
| 608 |
+
"terminal_next_frame": 554
|
| 609 |
+
},
|
| 610 |
+
{
|
| 611 |
+
"chunks": 71,
|
| 612 |
+
"demo": "demo_140",
|
| 613 |
+
"env_steps": 568,
|
| 614 |
+
"frames": 572,
|
| 615 |
+
"terminal_next_frame": 568
|
| 616 |
+
},
|
| 617 |
+
{
|
| 618 |
+
"chunks": 45,
|
| 619 |
+
"demo": "demo_141",
|
| 620 |
+
"env_steps": 357,
|
| 621 |
+
"frames": 361,
|
| 622 |
+
"terminal_next_frame": 357
|
| 623 |
+
},
|
| 624 |
+
{
|
| 625 |
+
"chunks": 48,
|
| 626 |
+
"demo": "demo_145",
|
| 627 |
+
"env_steps": 379,
|
| 628 |
+
"frames": 384,
|
| 629 |
+
"terminal_next_frame": 379
|
| 630 |
+
},
|
| 631 |
+
{
|
| 632 |
+
"chunks": 41,
|
| 633 |
+
"demo": "demo_147",
|
| 634 |
+
"env_steps": 328,
|
| 635 |
+
"frames": 333,
|
| 636 |
+
"terminal_next_frame": 328
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"chunks": 59,
|
| 640 |
+
"demo": "demo_149",
|
| 641 |
+
"env_steps": 466,
|
| 642 |
+
"frames": 474,
|
| 643 |
+
"terminal_next_frame": 466
|
| 644 |
+
},
|
| 645 |
+
{
|
| 646 |
+
"chunks": 87,
|
| 647 |
+
"demo": "demo_151",
|
| 648 |
+
"env_steps": 694,
|
| 649 |
+
"frames": 698,
|
| 650 |
+
"terminal_next_frame": 694
|
| 651 |
+
},
|
| 652 |
+
{
|
| 653 |
+
"chunks": 93,
|
| 654 |
+
"demo": "demo_152",
|
| 655 |
+
"env_steps": 740,
|
| 656 |
+
"frames": 744,
|
| 657 |
+
"terminal_next_frame": 740
|
| 658 |
+
},
|
| 659 |
+
{
|
| 660 |
+
"chunks": 50,
|
| 661 |
+
"demo": "demo_154",
|
| 662 |
+
"env_steps": 393,
|
| 663 |
+
"frames": 397,
|
| 664 |
+
"terminal_next_frame": 393
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"chunks": 85,
|
| 668 |
+
"demo": "demo_156",
|
| 669 |
+
"env_steps": 678,
|
| 670 |
+
"frames": 682,
|
| 671 |
+
"terminal_next_frame": 678
|
| 672 |
+
},
|
| 673 |
+
{
|
| 674 |
+
"chunks": 55,
|
| 675 |
+
"demo": "demo_157",
|
| 676 |
+
"env_steps": 433,
|
| 677 |
+
"frames": 437,
|
| 678 |
+
"terminal_next_frame": 433
|
| 679 |
+
},
|
| 680 |
+
{
|
| 681 |
+
"chunks": 57,
|
| 682 |
+
"demo": "demo_159",
|
| 683 |
+
"env_steps": 456,
|
| 684 |
+
"frames": 465,
|
| 685 |
+
"terminal_next_frame": 456
|
| 686 |
+
},
|
| 687 |
+
{
|
| 688 |
+
"chunks": 53,
|
| 689 |
+
"demo": "demo_161",
|
| 690 |
+
"env_steps": 421,
|
| 691 |
+
"frames": 425,
|
| 692 |
+
"terminal_next_frame": 421
|
| 693 |
+
},
|
| 694 |
+
{
|
| 695 |
+
"chunks": 59,
|
| 696 |
+
"demo": "demo_163",
|
| 697 |
+
"env_steps": 466,
|
| 698 |
+
"frames": 469,
|
| 699 |
+
"terminal_next_frame": 466
|
| 700 |
+
},
|
| 701 |
+
{
|
| 702 |
+
"chunks": 67,
|
| 703 |
+
"demo": "demo_164",
|
| 704 |
+
"env_steps": 529,
|
| 705 |
+
"frames": 533,
|
| 706 |
+
"terminal_next_frame": 529
|
| 707 |
+
},
|
| 708 |
+
{
|
| 709 |
+
"chunks": 68,
|
| 710 |
+
"demo": "demo_171",
|
| 711 |
+
"env_steps": 537,
|
| 712 |
+
"frames": 541,
|
| 713 |
+
"terminal_next_frame": 537
|
| 714 |
+
},
|
| 715 |
+
{
|
| 716 |
+
"chunks": 69,
|
| 717 |
+
"demo": "demo_172",
|
| 718 |
+
"env_steps": 550,
|
| 719 |
+
"frames": 554,
|
| 720 |
+
"terminal_next_frame": 550
|
| 721 |
+
},
|
| 722 |
+
{
|
| 723 |
+
"chunks": 54,
|
| 724 |
+
"demo": "demo_173",
|
| 725 |
+
"env_steps": 432,
|
| 726 |
+
"frames": 441,
|
| 727 |
+
"terminal_next_frame": 432
|
| 728 |
+
},
|
| 729 |
+
{
|
| 730 |
+
"chunks": 56,
|
| 731 |
+
"demo": "demo_174",
|
| 732 |
+
"env_steps": 448,
|
| 733 |
+
"frames": 452,
|
| 734 |
+
"terminal_next_frame": 448
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"chunks": 65,
|
| 738 |
+
"demo": "demo_175",
|
| 739 |
+
"env_steps": 515,
|
| 740 |
+
"frames": 525,
|
| 741 |
+
"terminal_next_frame": 515
|
| 742 |
+
},
|
| 743 |
+
{
|
| 744 |
+
"chunks": 49,
|
| 745 |
+
"demo": "demo_176",
|
| 746 |
+
"env_steps": 391,
|
| 747 |
+
"frames": 395,
|
| 748 |
+
"terminal_next_frame": 391
|
| 749 |
+
},
|
| 750 |
+
{
|
| 751 |
+
"chunks": 54,
|
| 752 |
+
"demo": "demo_177",
|
| 753 |
+
"env_steps": 429,
|
| 754 |
+
"frames": 433,
|
| 755 |
+
"terminal_next_frame": 429
|
| 756 |
+
},
|
| 757 |
+
{
|
| 758 |
+
"chunks": 57,
|
| 759 |
+
"demo": "demo_179",
|
| 760 |
+
"env_steps": 449,
|
| 761 |
+
"frames": 454,
|
| 762 |
+
"terminal_next_frame": 449
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"chunks": 53,
|
| 766 |
+
"demo": "demo_181",
|
| 767 |
+
"env_steps": 422,
|
| 768 |
+
"frames": 426,
|
| 769 |
+
"terminal_next_frame": 422
|
| 770 |
+
},
|
| 771 |
+
{
|
| 772 |
+
"chunks": 73,
|
| 773 |
+
"demo": "demo_182",
|
| 774 |
+
"env_steps": 582,
|
| 775 |
+
"frames": 586,
|
| 776 |
+
"terminal_next_frame": 582
|
| 777 |
+
},
|
| 778 |
+
{
|
| 779 |
+
"chunks": 56,
|
| 780 |
+
"demo": "demo_183",
|
| 781 |
+
"env_steps": 443,
|
| 782 |
+
"frames": 447,
|
| 783 |
+
"terminal_next_frame": 443
|
| 784 |
+
},
|
| 785 |
+
{
|
| 786 |
+
"chunks": 56,
|
| 787 |
+
"demo": "demo_186",
|
| 788 |
+
"env_steps": 447,
|
| 789 |
+
"frames": 451,
|
| 790 |
+
"terminal_next_frame": 447
|
| 791 |
+
},
|
| 792 |
+
{
|
| 793 |
+
"chunks": 89,
|
| 794 |
+
"demo": "demo_188",
|
| 795 |
+
"env_steps": 705,
|
| 796 |
+
"frames": 709,
|
| 797 |
+
"terminal_next_frame": 705
|
| 798 |
+
},
|
| 799 |
+
{
|
| 800 |
+
"chunks": 52,
|
| 801 |
+
"demo": "demo_190",
|
| 802 |
+
"env_steps": 416,
|
| 803 |
+
"frames": 421,
|
| 804 |
+
"terminal_next_frame": 416
|
| 805 |
+
},
|
| 806 |
+
{
|
| 807 |
+
"chunks": 52,
|
| 808 |
+
"demo": "demo_191",
|
| 809 |
+
"env_steps": 411,
|
| 810 |
+
"frames": 415,
|
| 811 |
+
"terminal_next_frame": 411
|
| 812 |
+
},
|
| 813 |
+
{
|
| 814 |
+
"chunks": 52,
|
| 815 |
+
"demo": "demo_192",
|
| 816 |
+
"env_steps": 413,
|
| 817 |
+
"frames": 416,
|
| 818 |
+
"terminal_next_frame": 413
|
| 819 |
+
},
|
| 820 |
+
{
|
| 821 |
+
"chunks": 62,
|
| 822 |
+
"demo": "demo_193",
|
| 823 |
+
"env_steps": 494,
|
| 824 |
+
"frames": 498,
|
| 825 |
+
"terminal_next_frame": 494
|
| 826 |
+
},
|
| 827 |
+
{
|
| 828 |
+
"chunks": 53,
|
| 829 |
+
"demo": "demo_196",
|
| 830 |
+
"env_steps": 419,
|
| 831 |
+
"frames": 423,
|
| 832 |
+
"terminal_next_frame": 419
|
| 833 |
+
},
|
| 834 |
+
{
|
| 835 |
+
"chunks": 59,
|
| 836 |
+
"demo": "demo_199",
|
| 837 |
+
"env_steps": 469,
|
| 838 |
+
"frames": 473,
|
| 839 |
+
"terminal_next_frame": 469
|
| 840 |
+
}
|
| 841 |
+
],
|
| 842 |
+
"pixel_cache": "/ssd/WorkSpace/RL-Postrain/data/pixels/tool_hang/v1",
|
| 843 |
+
"raw_coords": 342888,
|
| 844 |
+
"raw_coords_clipped": 0
|
| 845 |
+
},
|
| 846 |
+
"stats_hash": "919708648ac0650eea14b505ff433b77c7affe19a4514c64f3f32d6458412715"
|
| 847 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/resolved_config.json
ADDED
|
@@ -0,0 +1,1195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"decision_ref": "docs/scope.md [0928-I], [0930-S2DEC]",
|
| 3 |
+
"env": {
|
| 4 |
+
"chunk_length": 8,
|
| 5 |
+
"n_envs": 4
|
| 6 |
+
},
|
| 7 |
+
"learner": {
|
| 8 |
+
"convention": "unified_env_step",
|
| 9 |
+
"gamma_env": 0.999
|
| 10 |
+
},
|
| 11 |
+
"overrides": [
|
| 12 |
+
{
|
| 13 |
+
"decision_ref": "[1020-S2TOOLHANG5] configs/study2_protocol.yaml",
|
| 14 |
+
"key": "n_step_chunks",
|
| 15 |
+
"reason": "task-specific unified Study2 setting",
|
| 16 |
+
"upstream": 3,
|
| 17 |
+
"value": 5
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 21 |
+
"key": "offline.cohort_size",
|
| 22 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 23 |
+
"upstream": 50,
|
| 24 |
+
"value": 100
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"decision_ref": "[1023-OFFLINE100] user 2026-10-23: 100-demo BC -> all 100 demos as offline D_demo",
|
| 28 |
+
"key": "offline.episodes",
|
| 29 |
+
"reason": "tool_hang regime diverse100 BC cohort has 100 demos; the offline demo buffer uses the checkpoint's entire BC cohort",
|
| 30 |
+
"upstream": 50,
|
| 31 |
+
"value": 100
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"decision_ref": "[0930-S2DEC]",
|
| 35 |
+
"key": "learner.qc.agent.act_bounds_json",
|
| 36 |
+
"reason": "per-dimension normalized images of raw -1/+1 from the checkpoint action normalizer; clips every final action and Best-of-N candidate per dimension",
|
| 37 |
+
"upstream": null,
|
| 38 |
+
"value": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"decision_ref": "[0930-S2DEC]",
|
| 42 |
+
"key": "learner.qc.agent.act_max",
|
| 43 |
+
"reason": "compatibility scalar: maximum over dimensions of the per-dimension upper bounds (act_bounds_json is what clips)",
|
| 44 |
+
"upstream": 1.0,
|
| 45 |
+
"value": 3.0058202743530273
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"decision_ref": "[0930-S2DEC]",
|
| 49 |
+
"key": "learner.qc.agent.act_min",
|
| 50 |
+
"reason": "compatibility scalar: minimum over dimensions of the per-dimension lower bounds (act_bounds_json is what clips)",
|
| 51 |
+
"upstream": -1.0,
|
| 52 |
+
"value": -2.9472272396087646
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"decision_ref": "[0930-S2DEC]",
|
| 56 |
+
"key": "learner.qc.agent.actor_lr_partition",
|
| 57 |
+
"reason": "unified.lr.actor peak; outer env-step scale on the actor partition",
|
| 58 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 59 |
+
"value": 0.0001
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 63 |
+
"key": "learner.qc.agent.actor_num_samples",
|
| 64 |
+
"reason": "single BoN knob including QC TD",
|
| 65 |
+
"upstream": 32,
|
| 66 |
+
"value": 16
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 70 |
+
"key": "learner.qc.agent.best_of_n",
|
| 71 |
+
"reason": "single BoN knob: collection/eval/QC TD",
|
| 72 |
+
"upstream": 8,
|
| 73 |
+
"value": 16
|
| 74 |
+
},
|
| 75 |
+
{
|
| 76 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 77 |
+
"key": "learner.qc.agent.bon_agg",
|
| 78 |
+
"reason": "Best-of-N selection by the ensemble mean",
|
| 79 |
+
"upstream": null,
|
| 80 |
+
"value": "mean"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 84 |
+
"key": "learner.qc.agent.bon_sft",
|
| 85 |
+
"reason": "variant qc flag bon_sft",
|
| 86 |
+
"upstream": "upstream_yaml",
|
| 87 |
+
"value": false
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"decision_ref": "[0930-S2DEC]",
|
| 91 |
+
"key": "learner.qc.agent.critic_lr_partition",
|
| 92 |
+
"reason": "unified.lr.critic peak; outer env-step scale on the critic partition",
|
| 93 |
+
"upstream": "lr=0.0003 shared by actor and critic",
|
| 94 |
+
"value": 0.0003
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 98 |
+
"key": "learner.qc.agent.partition_optimizer",
|
| 99 |
+
"reason": "separate actor/critic optimizer partitions for the exact 2:1 critic:actor schedule (critic-only steps leave the actor untouched); each partition has its own clip_by_global_norm and Adam moments/step count (upstream: one global clip and one Adam count over all parameters)",
|
| 100 |
+
"upstream": null,
|
| 101 |
+
"value": true
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 105 |
+
"key": "learner.qc.agent.q_target_agg",
|
| 106 |
+
"reason": "REDQ-style TD target: min over a random critic subset",
|
| 107 |
+
"upstream": null,
|
| 108 |
+
"value": "subset_min"
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 112 |
+
"key": "learner.qc.agent.q_target_replace",
|
| 113 |
+
"reason": "subset drawn without replacement",
|
| 114 |
+
"upstream": null,
|
| 115 |
+
"value": false
|
| 116 |
+
},
|
| 117 |
+
{
|
| 118 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 119 |
+
"key": "learner.qc.agent.q_target_subset",
|
| 120 |
+
"reason": "TD-target subset size",
|
| 121 |
+
"upstream": null,
|
| 122 |
+
"value": 2
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"decision_ref": "[0930-S2DEC]",
|
| 126 |
+
"key": "learner.qc.agent.tau",
|
| 127 |
+
"reason": "unified.critic_target_tau: critic target polyak after every critic gradient step",
|
| 128 |
+
"upstream": 0.05,
|
| 129 |
+
"value": 0.005
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"decision_ref": "[0930-S2DEC]",
|
| 133 |
+
"key": "learner.qc.agent.value_hidden_dims",
|
| 134 |
+
"reason": "unified.critic_hidden (critics only; actors native)",
|
| 135 |
+
"upstream": [
|
| 136 |
+
512,
|
| 137 |
+
512,
|
| 138 |
+
512,
|
| 139 |
+
512,
|
| 140 |
+
512
|
| 141 |
+
],
|
| 142 |
+
"value": [
|
| 143 |
+
512,
|
| 144 |
+
512,
|
| 145 |
+
512
|
| 146 |
+
]
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 150 |
+
"key": "learner.qc.training.bc_pi_steps",
|
| 151 |
+
"reason": "Study1 pixel_state checkpoint is the BC policy; no offline BC phase",
|
| 152 |
+
"upstream": 1000000,
|
| 153 |
+
"value": 0
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 157 |
+
"key": "learner.qc.loop.buffer.utd_online",
|
| 158 |
+
"reason": "UTD=3 per executed action chunk",
|
| 159 |
+
"upstream": 1,
|
| 160 |
+
"value": "3 gradient updates per executed chunk"
|
| 161 |
+
},
|
| 162 |
+
{
|
| 163 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 164 |
+
"key": "learner.qc.loop.n_step",
|
| 165 |
+
"reason": "n-step=3 counted in action chunks",
|
| 166 |
+
"upstream": "1 chunk (sequence_length = horizon_length)",
|
| 167 |
+
"value": "5 executed chunks (unified_env_step window)"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 171 |
+
"key": "learner.qc.loop.critic_actor_ratio",
|
| 172 |
+
"reason": "critic every update, actor every 2nd global update",
|
| 173 |
+
"upstream": "1:1 (utd_q = utd_pi)",
|
| 174 |
+
"value": "2:1"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 178 |
+
"key": "learner.qc.loop.buffer.start_training",
|
| 179 |
+
"reason": "[1006-S2V5] offline demo buffer replaces the BC start rollout",
|
| 180 |
+
"upstream": 10000,
|
| 181 |
+
"value": "0: RL collects from step 0 (no warm-up); updates start after the first 1000-chunk round"
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 185 |
+
"key": "learner.qc.loop.buffer.offline_ratio",
|
| 186 |
+
"reason": "[1017-S2OFFBC] adaptive_ratio off",
|
| 187 |
+
"upstream": 0.0,
|
| 188 |
+
"value": "D_demo (the checkpoint's own BC cohort) kept in the replay for the whole run, online data FIFO; each batch uniform over D_demo + online windows"
|
| 189 |
+
},
|
| 190 |
+
{
|
| 191 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 192 |
+
"key": "learner.qc.loop.buffer.size",
|
| 193 |
+
"reason": "one shared replay of executed chunks; success buffer = mask",
|
| 194 |
+
"upstream": 2000000,
|
| 195 |
+
"value": "1000000 executed chunks, shared FIFO"
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 199 |
+
"key": "learner.qc.actor_architecture",
|
| 200 |
+
"reason": "Study2 fine-tunes the Study1 pixel_state CFM",
|
| 201 |
+
"upstream": [
|
| 202 |
+
512,
|
| 203 |
+
512,
|
| 204 |
+
512,
|
| 205 |
+
512,
|
| 206 |
+
512
|
| 207 |
+
],
|
| 208 |
+
"value": "checkpoint StandardGenerator (EMA generator copy)"
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"decision_ref": "docs/scope.md [0928-I]",
|
| 212 |
+
"key": "learner.qc.observation",
|
| 213 |
+
"reason": "Study2 runs on pixel_state checkpoints with frozen Ri",
|
| 214 |
+
"upstream": "low-dim state",
|
| 215 |
+
"value": "frozen Ri features c [137] from the checkpoint encoder"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"decision_ref": "[0930-S2DEC]",
|
| 219 |
+
"key": "learner.qc.reward",
|
| 220 |
+
"reason": "unified.reward = binary_first_success (no -1 step penalty, no +5 success bonus)",
|
| 221 |
+
"upstream": "native OGPO robomimic wrapper reward: -1 per non-success primitive step, +5 on the terminating success step (post_success_steps=0)",
|
| 222 |
+
"value": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged"
|
| 223 |
+
},
|
| 224 |
+
{
|
| 225 |
+
"decision_ref": "[0930-S2DEC]",
|
| 226 |
+
"key": "learner.qc.td_target",
|
| 227 |
+
"reason": "one TD convention for every learner; unified.timeout_bootstrap = false",
|
| 228 |
+
"upstream": "rewards/masks of one chunk (sequence_length = horizon), timeout bootstraps (masks = 1 - terminated)",
|
| 229 |
+
"value": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk"
|
| 230 |
+
},
|
| 231 |
+
{
|
| 232 |
+
"decision_ref": "[0930-S2DEC]",
|
| 233 |
+
"key": "learner.qc.eval",
|
| 234 |
+
"reason": "decision 7: Best-of-N main evaluation, single entry eval.best_of_n",
|
| 235 |
+
"upstream": "compute_flow_actions (online BC flow actor, Euler ODE), no Best-of-N",
|
| 236 |
+
"value": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise"
|
| 237 |
+
},
|
| 238 |
+
{
|
| 239 |
+
"decision_ref": "[1002-S2ALIGN]",
|
| 240 |
+
"key": "learner.qc.lr_schedule",
|
| 241 |
+
"reason": "RL env-step clock shared with extra success TD",
|
| 242 |
+
"upstream": "native per-optimizer schedule",
|
| 243 |
+
"value": {
|
| 244 |
+
"actor": 0.0001,
|
| 245 |
+
"critic": 0.0003,
|
| 246 |
+
"decay_end_env_steps": 1606196,
|
| 247 |
+
"min_ratio": 0.1,
|
| 248 |
+
"schedule": "warmup_cosine",
|
| 249 |
+
"temperature": 0.0003,
|
| 250 |
+
"warmup_env_steps": 20000
|
| 251 |
+
}
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"decision_ref": "[0930-S2DEC]",
|
| 255 |
+
"key": "learner.qc.agent.q_min_q_max",
|
| 256 |
+
"reason": "Q range of the binary first-success reward (upstream get_q_bounds assumes the -1 per-step reward); used only by hlgauss critics",
|
| 257 |
+
"upstream": [
|
| 258 |
+
-999.9999999999991,
|
| 259 |
+
0.0
|
| 260 |
+
],
|
| 261 |
+
"value": [
|
| 262 |
+
0.0,
|
| 263 |
+
1.0
|
| 264 |
+
]
|
| 265 |
+
}
|
| 266 |
+
],
|
| 267 |
+
"protocol": {
|
| 268 |
+
"checkpoint_every_rounds": 5,
|
| 269 |
+
"checkpoint_retention": {
|
| 270 |
+
"best": true,
|
| 271 |
+
"collapse_drop": {
|
| 272 |
+
"min_abs": 0.2,
|
| 273 |
+
"ratio": 0.5
|
| 274 |
+
},
|
| 275 |
+
"diagnostics": true,
|
| 276 |
+
"every_evals": 3
|
| 277 |
+
},
|
| 278 |
+
"collection": {
|
| 279 |
+
"n_envs": 4,
|
| 280 |
+
"train_seed_base": 100000000,
|
| 281 |
+
"train_seed_stride": 1000000,
|
| 282 |
+
"training_freq_chunks": 1000
|
| 283 |
+
},
|
| 284 |
+
"convergence": {
|
| 285 |
+
"early_stop": false,
|
| 286 |
+
"gain_lag": 10,
|
| 287 |
+
"high_sr": 0.95,
|
| 288 |
+
"high_sr_evals": 5,
|
| 289 |
+
"ma_window": 10,
|
| 290 |
+
"max_env_steps": 1606196,
|
| 291 |
+
"min_env_steps": 200000,
|
| 292 |
+
"min_gain": 0.02,
|
| 293 |
+
"zero_success_evals": 8
|
| 294 |
+
},
|
| 295 |
+
"critic_ensemble": {
|
| 296 |
+
"num": 10,
|
| 297 |
+
"policy": "mean",
|
| 298 |
+
"target": {
|
| 299 |
+
"kind": "subset_min",
|
| 300 |
+
"replace": false,
|
| 301 |
+
"size": 2
|
| 302 |
+
}
|
| 303 |
+
},
|
| 304 |
+
"eval": {
|
| 305 |
+
"best_of_n": 16,
|
| 306 |
+
"episodes": 200,
|
| 307 |
+
"every_env_steps": 20000,
|
| 308 |
+
"include_step0": true,
|
| 309 |
+
"seed_start": 960000,
|
| 310 |
+
"workers": 4
|
| 311 |
+
},
|
| 312 |
+
"final_eval": {
|
| 313 |
+
"episodes": 300,
|
| 314 |
+
"on_zero_stop": true,
|
| 315 |
+
"seed_start": 800000
|
| 316 |
+
},
|
| 317 |
+
"n_step_chunks": 5,
|
| 318 |
+
"offline": {
|
| 319 |
+
"adaptive_ratio": false,
|
| 320 |
+
"cohort_size": 100,
|
| 321 |
+
"episodes": 100,
|
| 322 |
+
"learners_without_offline": [
|
| 323 |
+
"dsrl_sac"
|
| 324 |
+
],
|
| 325 |
+
"ratio": {
|
| 326 |
+
"decay_env_steps": {
|
| 327 |
+
"can": 160000,
|
| 328 |
+
"square": 320000,
|
| 329 |
+
"tool_hang": 640000,
|
| 330 |
+
"transport": 320000
|
| 331 |
+
},
|
| 332 |
+
"end": 0.1,
|
| 333 |
+
"start": {
|
| 334 |
+
"can": 0.7,
|
| 335 |
+
"square": 0.5,
|
| 336 |
+
"tool_hang": 0.5,
|
| 337 |
+
"transport": 0.9
|
| 338 |
+
}
|
| 339 |
+
},
|
| 340 |
+
"selection": "ascending_demo_index",
|
| 341 |
+
"source": "bc_regime",
|
| 342 |
+
"unsupported_learners": [
|
| 343 |
+
"zprl"
|
| 344 |
+
]
|
| 345 |
+
},
|
| 346 |
+
"replay": {
|
| 347 |
+
"capacity_chunks": 1000000,
|
| 348 |
+
"pin_start_successes": false,
|
| 349 |
+
"success_buffer": "success_mask_on_shared_fifo",
|
| 350 |
+
"success_skip_rule": "skip_below_success_batch_size",
|
| 351 |
+
"unit": "executed_chunk"
|
| 352 |
+
},
|
| 353 |
+
"start": {
|
| 354 |
+
"max_episodes": 200,
|
| 355 |
+
"mode": "offline_buffer",
|
| 356 |
+
"policy": {
|
| 357 |
+
"default": "base_cfm_ode",
|
| 358 |
+
"zprl": "vib_base"
|
| 359 |
+
},
|
| 360 |
+
"target_successes": 20
|
| 361 |
+
},
|
| 362 |
+
"unified": {
|
| 363 |
+
"actor_target_tau": 0.005,
|
| 364 |
+
"critic_hidden": [
|
| 365 |
+
512,
|
| 366 |
+
512,
|
| 367 |
+
512
|
| 368 |
+
],
|
| 369 |
+
"critic_target_every_critic_step": true,
|
| 370 |
+
"critic_target_tau": 0.005,
|
| 371 |
+
"dsrl_noise_imitation_reference_dims": 28,
|
| 372 |
+
"gamma_env": 0.999,
|
| 373 |
+
"lr": {
|
| 374 |
+
"actor": 0.0001,
|
| 375 |
+
"critic": 0.0003,
|
| 376 |
+
"decay_end_env_steps": 1606196,
|
| 377 |
+
"min_ratio": 0.1,
|
| 378 |
+
"schedule": "warmup_cosine",
|
| 379 |
+
"temperature": 0.0003,
|
| 380 |
+
"warmup_env_steps": 20000
|
| 381 |
+
},
|
| 382 |
+
"reward": "binary_first_success",
|
| 383 |
+
"success_td_polyak": true,
|
| 384 |
+
"timeout_bootstrap": false,
|
| 385 |
+
"zprl_imitation": "mse_pretanh_mean"
|
| 386 |
+
},
|
| 387 |
+
"updates": {
|
| 388 |
+
"batch_size": 256,
|
| 389 |
+
"critic_per_actor": 2,
|
| 390 |
+
"success_batch_size": 256,
|
| 391 |
+
"utd_per_chunk": 3
|
| 392 |
+
},
|
| 393 |
+
"version": "study2_protocol_v10"
|
| 394 |
+
},
|
| 395 |
+
"protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
|
| 396 |
+
"provenance_extra": {
|
| 397 |
+
"agent_library": {
|
| 398 |
+
"framework": "jax",
|
| 399 |
+
"sources": {
|
| 400 |
+
"__init__.py": "0356b394bbae9d0738649e4864a034c4ee5d6f329c8a7f862717b3428e52313c",
|
| 401 |
+
"_vendor/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 402 |
+
"_vendor/ogpo/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 403 |
+
"_vendor/ogpo/agents/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 404 |
+
"_vendor/ogpo/agents/modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 405 |
+
"_vendor/ogpo/agents/modules/bc_helper.py": "8913392cc0434057323b2f5b28aaeb36d2c31e6b129ef09d21352353fe276c14",
|
| 406 |
+
"_vendor/ogpo/agents/modules/dist_helper.py": "c0f429c6c70082f5f5258ffffdf4bf0e6b2e1587c26528d3ee937a4e302d9951",
|
| 407 |
+
"_vendor/ogpo/agents/modules/flax_utils.py": "36f27885a086a4ae561171de4650da86330dcd68f17694e3f71278677c614037",
|
| 408 |
+
"_vendor/ogpo/agents/modules/igp_targets_helper.py": "b0343a236a856ec734cc73a1e73186d7d6520dd9efc34b3597327d21b55c9522",
|
| 409 |
+
"_vendor/ogpo/agents/modules/maintenance.py": "7b40d670c277783e118a0669f57aa10370c76b7b8fe3c1d27d16af1a3e9f0865",
|
| 410 |
+
"_vendor/ogpo/agents/modules/pg_helper.py": "cb2e0e83e0c95af408c9bac86fc9ccf1d7e427b3edb315c5d042de5d0a9328d4",
|
| 411 |
+
"_vendor/ogpo/agents/modules/q_helper.py": "71d874b74af5d724d82bd9c4a0a6027d8795e5dc26755ae7838abc3cda54718e",
|
| 412 |
+
"_vendor/ogpo/agents/ogpo.py": "3393282fd4a972c97951715a5776a506e37102cbe5e9c21e5e1ed1e6288696b2",
|
| 413 |
+
"_vendor/ogpo/agents/qc.py": "55906d8b598ce0a2607782e886697c3766dd281a0664ec05bf96173fc8c22c56",
|
| 414 |
+
"_vendor/ogpo/networks/__init__.py": "089130459e91d2b43f5bca45e2659507379fe62351ce6d2d91a49a0387ed2e90",
|
| 415 |
+
"_vendor/ogpo/networks/actors.py": "c0ad2b12439f5da82128e35907314741c22ccc70692de33fed90e67a32f8d86a",
|
| 416 |
+
"_vendor/ogpo/networks/actors_fql.py": "0d9f8646f1230b442bc80457252656c007431ca62458afa5fdecd15150ed9c4d",
|
| 417 |
+
"_vendor/ogpo/networks/critics.py": "77e40f7ea9d6837f8f89d889bfcd7a70e0ef20076f65720d79a9bef3db5130b4",
|
| 418 |
+
"_vendor/ogpo/networks/encoders.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
| 419 |
+
"_vendor/ogpo/networks/encoders/__init__.py": "41ae7973f560af5539cff23643843a0c56c253722bb17d7372a9ade029abb670",
|
| 420 |
+
"_vendor/ogpo/networks/encoders/impala.py": "62b7f1dab0aefe6662d3fa0fc59417d125cb9687af71d7875d533e24ac3e89d2",
|
| 421 |
+
"_vendor/ogpo/networks/encoders/minvit.py": "93f0af908450588cdbc9fc18929247b55447bb7972833388244707628328ed33",
|
| 422 |
+
"_vendor/ogpo/networks/encoders/paligemma.py": "0f2a448fd52a2c161f921e7fd7d79be462f98db34c09bf68877f8ed67de9e82c",
|
| 423 |
+
"_vendor/ogpo/networks/encoders/vision_proprio.py": "01ade56e774aeb29c3e7cd1d59eff5e43b7827d102769a450f8d7cc5a17666c8",
|
| 424 |
+
"_vendor/ogpo/networks/modules/__init__.py": "39a4fe9455c1b58c505fa1d9c0a90179dd64cdd459e8341d9ef7fcf6b7af9aae",
|
| 425 |
+
"_vendor/ogpo/networks/modules/attention.py": "a835876c6269ba0c5c63f49005027e5524a750f0c6e7af8697093b9bb8ff4c55",
|
| 426 |
+
"_vendor/ogpo/networks/modules/distributions.py": "5430f1bc6ee7fa71aac65664c9c3f13500d1fec3c5b2b1d80fc6590cca8ac71d",
|
| 427 |
+
"_vendor/ogpo/networks/modules/misc.py": "527793d4e64be2bbc8c22e73a574d5cb604e4672eabd0db9843853107502f620",
|
| 428 |
+
"_vendor/ogpo/networks/modules/mlp.py": "e2cf4d9923d95df8747c1554cf94f20000e108b1a9b0d9b0729a759b597a1316",
|
| 429 |
+
"_vendor/ogpo/networks/modules/time_embedding.py": "54d1a400b1b179a75c5aa55501031aa5d6513da12715ab75b89eaacdbe9f76e1",
|
| 430 |
+
"_vendor/ogpo/networks/modules/two_tier.py": "aa5182a5932cd500f95f4fddcd16429d66c968988639ddf3d6b33a2297953166",
|
| 431 |
+
"dice.py": "6989a71a178af904061e418a94c06615acb7576d9d32548566cc0032e273f22b",
|
| 432 |
+
"dsrl_na.py": "973970bfb02b87ad995f04572bc4d8443cc1e92d6447fbcbcfb80d8fa97cd9ce",
|
| 433 |
+
"dsrl_sac.py": "292d55b2d6f314ec12d17c77c2da2badef6a3fbdde89ba953044e074da6fbc6c",
|
| 434 |
+
"factory.py": "0117708b7cdab69cfaa83bba70f1101aff7b33866117946c377a5cb2fcc6ea49",
|
| 435 |
+
"flow.py": "76dce89f145b4194885524731523eada50e86f8d06b86f7cfaffd0fb3e3b779a",
|
| 436 |
+
"native_config.py": "e9dbf33c0ec15a654c2dde7f504d4e2492c29cbdf84c4eedf42cda94c158e4de",
|
| 437 |
+
"native_flow.py": "e9a9dc59e4e2ccaa7355bebb3c98cc9c1e0080c493889d58bfefdc7bc341c763",
|
| 438 |
+
"native_replay.py": "4f5922a41c5f4e93c599794143c0a26d8b22cb6c38d675ede9538a7bd2b30402",
|
| 439 |
+
"native_residual.py": "00936a650eed103e380f201c9917a9bc414d7b48ba244d0018ebd7b032b3af67",
|
| 440 |
+
"native_train.py": "83bd25879b1096afcb8c1e7a4c34ab77c652dafde10d86f7fdea221c98f6d4a4",
|
| 441 |
+
"networks.py": "81460ac49a7b67ef66b0efd04e26e86c320456c190493143ea77b57c8fbc195d",
|
| 442 |
+
"ogpo.py": "3501a14eb1e1dd0e7194de541dc1af2dd0b4584f102b7e23b0b625d490c79093",
|
| 443 |
+
"optim.py": "41f5c0e4907e822449154959067a84f09820010bc02c6a01a74b5d5478a76192",
|
| 444 |
+
"qc.py": "38ef1b0ed7a4d0833ee50a182262a3bd668931d9499483fa58f7cad43b3f9274",
|
| 445 |
+
"registry.py": "0e7928dac526397186290fe1a914403daa69c39c2c9da9aa395bacfbca9dc617",
|
| 446 |
+
"residual.py": "2dfda94aee07537924912d14693b7d0a41a76c9aea02852a006e5f4877c082ee",
|
| 447 |
+
"telemetry.py": "e5716864ea9b2126b50b86fec78d56ecfd96f17832d5006f83cb2763e6092feb",
|
| 448 |
+
"train.py": "af62bcfe092ab39e79eae9bc1f297430865cd31f61212faf17863e2b6de64ad8",
|
| 449 |
+
"vib.py": "ab3a6721c7c051d21e7955940942bc0f6d064635dd9c73b7275589c150f19f5e",
|
| 450 |
+
"vib_workflow.py": "35b264975fbbcb7f1142af4284f245c9de043e3dab6689ca40657e7be47090f2",
|
| 451 |
+
"zprl.py": "fefe27f991d1a482a57b04738d5b22b9879f0ea405e1935e94dd58ba72c30609"
|
| 452 |
+
},
|
| 453 |
+
"version": "study2_jax_agents_v2"
|
| 454 |
+
},
|
| 455 |
+
"base_framework": "jax",
|
| 456 |
+
"bc_identity": {
|
| 457 |
+
"architecture": "dice_vision_resnet_flow_unet_v1",
|
| 458 |
+
"family": "StandardBaseBC",
|
| 459 |
+
"framework": "jax"
|
| 460 |
+
},
|
| 461 |
+
"checkpoint": "/ssd/WorkSpace/RL-Postrain/runs/operations/20261024_toolhang_diverse100_v10_three/base/diverse100/epoch_300.pkl",
|
| 462 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 463 |
+
"experiment": "tool_hang",
|
| 464 |
+
"frozen_hash": "719a7185a0d6a4877c76a5e1d9b1adb163a3d863d28bced54e79d534e8db6873",
|
| 465 |
+
"learner": {
|
| 466 |
+
"action_bounds": {
|
| 467 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 468 |
+
"compat_act_max": 3.0058202743530273,
|
| 469 |
+
"compat_act_min": -2.9472272396087646,
|
| 470 |
+
"hi": [
|
| 471 |
+
1.0,
|
| 472 |
+
1.0,
|
| 473 |
+
1.0,
|
| 474 |
+
3.0058202743530273,
|
| 475 |
+
1.8893057107925415,
|
| 476 |
+
1.0,
|
| 477 |
+
1.0
|
| 478 |
+
],
|
| 479 |
+
"kind": "per_dimension",
|
| 480 |
+
"lo": [
|
| 481 |
+
-1.0,
|
| 482 |
+
-1.0,
|
| 483 |
+
-1.0,
|
| 484 |
+
-2.9472272396087646,
|
| 485 |
+
-1.9721344709396362,
|
| 486 |
+
-1.318867564201355,
|
| 487 |
+
-1.0
|
| 488 |
+
],
|
| 489 |
+
"note": "[0930-S2DEC] decision 11: per-dimension normalized bounds lo_d=(-1-offset_d)/scale_d, hi_d=(1-offset_d)/scale_d in agent.act_bounds_json (JSON string: the agent config stays hashable/comparable; pg_helper.action_bounds parses it once) replace the scalar final clip in every sampler, so Best-of-N candidates are clipped per dimension before Q scoring and selection and critics train on executed actions; scalar act_min/act_max = per-dimension extremes are kept for compatibility"
|
| 490 |
+
},
|
| 491 |
+
"agent_config": {
|
| 492 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 493 |
+
"act_max": 3.0058202743530273,
|
| 494 |
+
"act_min": -2.9472272396087646,
|
| 495 |
+
"action_chunking": true,
|
| 496 |
+
"action_critic_hidden_dims": null,
|
| 497 |
+
"action_dim": 7,
|
| 498 |
+
"action_num_qs": 10,
|
| 499 |
+
"action_repeat": false,
|
| 500 |
+
"actor_backbone": "mlp",
|
| 501 |
+
"actor_decay_steps": 50000,
|
| 502 |
+
"actor_end_value": 2e-05,
|
| 503 |
+
"actor_hidden_dims": [
|
| 504 |
+
512,
|
| 505 |
+
512,
|
| 506 |
+
512,
|
| 507 |
+
512,
|
| 508 |
+
512
|
| 509 |
+
],
|
| 510 |
+
"actor_layer_norm": false,
|
| 511 |
+
"actor_lr": 0.0003,
|
| 512 |
+
"actor_lr_partition": 0.0001,
|
| 513 |
+
"actor_num_samples": 16,
|
| 514 |
+
"actor_obs": "state",
|
| 515 |
+
"actor_scheduler": "cosine",
|
| 516 |
+
"actor_tau": 0.005,
|
| 517 |
+
"actor_type": "best-of-n",
|
| 518 |
+
"actor_warmup_steps": 2000,
|
| 519 |
+
"actor_weight_decay": 0.0,
|
| 520 |
+
"adv_clip_min": null,
|
| 521 |
+
"agent_name": "acfql",
|
| 522 |
+
"alpha": 500,
|
| 523 |
+
"batch_size": 256,
|
| 524 |
+
"best_of_n": 16,
|
| 525 |
+
"bon_agg": "mean",
|
| 526 |
+
"bon_sft": false,
|
| 527 |
+
"bon_sft_coeff": 1.0,
|
| 528 |
+
"bootstrap_ratio": 0.5,
|
| 529 |
+
"bptt_num_samples": 4,
|
| 530 |
+
"calql_q_subsample": 2,
|
| 531 |
+
"clip_grad_norm": 1000.0,
|
| 532 |
+
"clip_intermediate_actions": true,
|
| 533 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 534 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 535 |
+
"constant_noise_std": 0.01,
|
| 536 |
+
"cql_alpha": 5.0,
|
| 537 |
+
"cql_n_actions": 10,
|
| 538 |
+
"cql_temp": 1.0,
|
| 539 |
+
"critic_backbone": "mlp",
|
| 540 |
+
"critic_decay_steps": 5000,
|
| 541 |
+
"critic_end_value": 1e-08,
|
| 542 |
+
"critic_loss_type": "mse",
|
| 543 |
+
"critic_lr": 0.0003,
|
| 544 |
+
"critic_lr_partition": 0.0003,
|
| 545 |
+
"critic_obs": "state",
|
| 546 |
+
"critic_scheduler": "constant",
|
| 547 |
+
"critic_warmup_steps": 500,
|
| 548 |
+
"critic_weight_decay": 1e-05,
|
| 549 |
+
"denoised_clip_value": 1.0,
|
| 550 |
+
"discount": 0.999,
|
| 551 |
+
"edit_action_scale": 0.5,
|
| 552 |
+
"edit_actor_hidden_dims": null,
|
| 553 |
+
"edit_hidden_dims": [
|
| 554 |
+
256,
|
| 555 |
+
256,
|
| 556 |
+
256
|
| 557 |
+
],
|
| 558 |
+
"edit_init_temperature": 1.0,
|
| 559 |
+
"edit_layer_norm": true,
|
| 560 |
+
"edit_lr": 0.0003,
|
| 561 |
+
"edit_policy_bound": 0.1,
|
| 562 |
+
"edit_start_step": 100000,
|
| 563 |
+
"edit_target_entropy": null,
|
| 564 |
+
"edit_weight_decay": 0.0,
|
| 565 |
+
"encoder": null,
|
| 566 |
+
"entropy_backup": false,
|
| 567 |
+
"flow_steps": 10,
|
| 568 |
+
"fql_distillation_coeff": 10.0,
|
| 569 |
+
"fql_train_main_policy": true,
|
| 570 |
+
"freeze_vision_encoder": false,
|
| 571 |
+
"ft_flow_steps": 10,
|
| 572 |
+
"horizon_length": 8,
|
| 573 |
+
"init_temperature": 1.0,
|
| 574 |
+
"layer_norm": true,
|
| 575 |
+
"lr": 0.0003,
|
| 576 |
+
"lr_action_critic": null,
|
| 577 |
+
"lr_actor": null,
|
| 578 |
+
"lr_actor_bc_flow": null,
|
| 579 |
+
"lr_critic": null,
|
| 580 |
+
"lr_edit_actor": null,
|
| 581 |
+
"lr_noise_critic": null,
|
| 582 |
+
"max_noise_std": 0.01,
|
| 583 |
+
"mc_regression": false,
|
| 584 |
+
"mc_regression_coeff": 1.0,
|
| 585 |
+
"min_noise_std": 0.01,
|
| 586 |
+
"mip_q_ensemble": false,
|
| 587 |
+
"mip_q_noise_scale": 1.0,
|
| 588 |
+
"mip_q_t_star": 0.9,
|
| 589 |
+
"mip_t_star": 0.999,
|
| 590 |
+
"muon_beta": 0.95,
|
| 591 |
+
"muon_nesterov": true,
|
| 592 |
+
"muon_ns_steps": 5,
|
| 593 |
+
"n_action_samples": 8,
|
| 594 |
+
"n_edit_samples": 8,
|
| 595 |
+
"noise_critic_grad_steps": 1,
|
| 596 |
+
"noise_dim": null,
|
| 597 |
+
"noise_net_hidden_dims": [
|
| 598 |
+
256,
|
| 599 |
+
256
|
| 600 |
+
],
|
| 601 |
+
"noise_net_layer_norm": true,
|
| 602 |
+
"normalize_act_space_dimension": false,
|
| 603 |
+
"normalize_denoising_horizon": false,
|
| 604 |
+
"normalize_q_loss": false,
|
| 605 |
+
"num_bins": 256,
|
| 606 |
+
"num_ensemble_members": 10,
|
| 607 |
+
"num_qs": 10,
|
| 608 |
+
"ob_dims": [
|
| 609 |
+
137
|
| 610 |
+
],
|
| 611 |
+
"one_step_decay_steps": null,
|
| 612 |
+
"one_step_end_value": null,
|
| 613 |
+
"one_step_hidden_dims": [
|
| 614 |
+
512,
|
| 615 |
+
512,
|
| 616 |
+
512,
|
| 617 |
+
512
|
| 618 |
+
],
|
| 619 |
+
"one_step_layer_norm": false,
|
| 620 |
+
"one_step_lr": null,
|
| 621 |
+
"one_step_scheduler": null,
|
| 622 |
+
"one_step_warmup_steps": null,
|
| 623 |
+
"optimizer_action_critic": null,
|
| 624 |
+
"optimizer_actor": "adam",
|
| 625 |
+
"optimizer_actor_bc_flow": "adam",
|
| 626 |
+
"optimizer_critic": "adam",
|
| 627 |
+
"optimizer_edit_actor": null,
|
| 628 |
+
"paligemma_tokenizer_path": null,
|
| 629 |
+
"partition_optimizer": true,
|
| 630 |
+
"policy_type": "flow",
|
| 631 |
+
"ppo_batch_size": 256,
|
| 632 |
+
"ppo_lr": 4.5e-05,
|
| 633 |
+
"proprio_dim": null,
|
| 634 |
+
"q_agg": "mean",
|
| 635 |
+
"q_max": 1.0,
|
| 636 |
+
"q_min": 0.0,
|
| 637 |
+
"q_target_agg": "subset_min",
|
| 638 |
+
"q_target_replace": false,
|
| 639 |
+
"q_target_subset": 2,
|
| 640 |
+
"q_variance_reduction": false,
|
| 641 |
+
"q_vr_num_samples": 8,
|
| 642 |
+
"q_vr_reduction": "mean",
|
| 643 |
+
"randn_clip_value": 3.0,
|
| 644 |
+
"shortcut_denoising_steps": 4,
|
| 645 |
+
"shortcut_inference_steps": 4,
|
| 646 |
+
"simba_actor_hidden_dims": [
|
| 647 |
+
1024,
|
| 648 |
+
1024
|
| 649 |
+
],
|
| 650 |
+
"simba_critic_hidden_dims": [
|
| 651 |
+
1024,
|
| 652 |
+
1024,
|
| 653 |
+
1024
|
| 654 |
+
],
|
| 655 |
+
"subsample_bon": false,
|
| 656 |
+
"target_entropy": null,
|
| 657 |
+
"tau": 0.005,
|
| 658 |
+
"tf_pi_conditioning": "adaln",
|
| 659 |
+
"tf_pi_dropout": 0.0,
|
| 660 |
+
"tf_pi_embed_dim": 256,
|
| 661 |
+
"tf_pi_heads": 4,
|
| 662 |
+
"tf_pi_layers": 8,
|
| 663 |
+
"tf_q_conditioning": "adaln",
|
| 664 |
+
"tf_q_dropout": 0.0,
|
| 665 |
+
"tf_q_embed_dim": 256,
|
| 666 |
+
"tf_q_heads": 4,
|
| 667 |
+
"tf_q_layers": 8,
|
| 668 |
+
"time_embedding": "sinusoidal",
|
| 669 |
+
"time_embedding_dim": 32,
|
| 670 |
+
"train_one_step_in_offline": true,
|
| 671 |
+
"train_one_step_in_online": true,
|
| 672 |
+
"useFiLM": false,
|
| 673 |
+
"useSimBa": false,
|
| 674 |
+
"useTF": false,
|
| 675 |
+
"use_constant_noise": true,
|
| 676 |
+
"use_constant_scheduler_for_bc": true,
|
| 677 |
+
"use_denoiser": false,
|
| 678 |
+
"use_edit_actor": false,
|
| 679 |
+
"use_mip_q": false,
|
| 680 |
+
"use_muon": false,
|
| 681 |
+
"use_one_step_policy": false,
|
| 682 |
+
"use_shortcut": false,
|
| 683 |
+
"use_state": "proprio",
|
| 684 |
+
"use_success_buffer_q": false,
|
| 685 |
+
"value_hidden_dims": [
|
| 686 |
+
512,
|
| 687 |
+
512,
|
| 688 |
+
512
|
| 689 |
+
],
|
| 690 |
+
"vit_encoder": "minvit",
|
| 691 |
+
"vit_img_proj_dim": 0,
|
| 692 |
+
"vit_state_proj_dim": 0
|
| 693 |
+
},
|
| 694 |
+
"convention": "unified_env_step",
|
| 695 |
+
"eval_rule": "Best-of-16 (protocol eval.best_of_n): per eval state 16 ODE candidates from N(0, I) noises, QC online `actor_bc_flow` via compute_flow_actions (Euler ODE); candidates clipped per dimension (act_bounds_json); scored by the ONLINE critic-ensemble mean (network 'critic'); the argmax candidate is executed (first on ties); no SDE noise, no exploration noise",
|
| 696 |
+
"gamma_env": 0.999,
|
| 697 |
+
"gamma_source": "protocol unified.gamma_env ([0930-S2DEC])",
|
| 698 |
+
"learner": "qc",
|
| 699 |
+
"optimizers": {
|
| 700 |
+
"actor_lr": 0.0001,
|
| 701 |
+
"critic_lr": 0.0003,
|
| 702 |
+
"reset_optimizers_with_lr": "not used",
|
| 703 |
+
"schedule": {
|
| 704 |
+
"actor": 0.0001,
|
| 705 |
+
"critic": 0.0003,
|
| 706 |
+
"decay_end_env_steps": 1606196,
|
| 707 |
+
"min_ratio": 0.1,
|
| 708 |
+
"schedule": "warmup_cosine",
|
| 709 |
+
"temperature": 0.0003,
|
| 710 |
+
"warmup_env_steps": 20000
|
| 711 |
+
}
|
| 712 |
+
},
|
| 713 |
+
"qc_partition_optimizer": "QC partition_optimizer: actor and critic partitions each run their own clip_by_global_norm(clip_grad_norm) and Adam (own moments and step count). Upstream uses one clip over all parameters and one Adam count; here the actor Adam count advances only on actor steps (like OGPO's separate actor/critic TrainStates)",
|
| 714 |
+
"recipe": {
|
| 715 |
+
"derived": {
|
| 716 |
+
"agent.actor_num_samples": {
|
| 717 |
+
"after": 8,
|
| 718 |
+
"before": 32
|
| 719 |
+
},
|
| 720 |
+
"agent.actor_type": {
|
| 721 |
+
"after": "best-of-n",
|
| 722 |
+
"before": "upstream_yaml"
|
| 723 |
+
},
|
| 724 |
+
"agent.bon_sft": {
|
| 725 |
+
"after": false,
|
| 726 |
+
"before": "upstream_yaml"
|
| 727 |
+
}
|
| 728 |
+
},
|
| 729 |
+
"explicit_overrides": {},
|
| 730 |
+
"source": "third_party/ogpo/scripts/qc/toolhang.sh",
|
| 731 |
+
"upstream_exact": false
|
| 732 |
+
},
|
| 733 |
+
"reward_mapping": "binary_first_success from the backbone: r=1 on the primitive step that first reaches success (the episode terminates there), 0 otherwise; the learner uses the backbone window reward unchanged",
|
| 734 |
+
"rollout_rule": "sample_actions_BON_with_requested (upstream sample_actions_BON plus the selected candidate's pre-clip action); Study2 rollouts require best_of_n > 1",
|
| 735 |
+
"scan_block": 100,
|
| 736 |
+
"start_rule": "base_cfm_ode: the PixelPolicy.sample Euler ODE (evaluate_pixel base) with fresh N(0,I) noise; executed = clip(requested, -1, 1), checked against PixelPolicy.from_noise (base_equivalence.json)",
|
| 737 |
+
"success_terms": {
|
| 738 |
+
"bc_on_success_batch": false,
|
| 739 |
+
"critic_success_td": false,
|
| 740 |
+
"missing_success_batch": "[0930-S2DEC] decision 9: when fewer eligible success windows than one success batch exist (backbone returns None) the update skips the success terms and counts it (success_missing). ogpo_plus: success_flag=False with agent.bc_missing_success='skip' -> no BC term (the PPO loss alone updates the actor) and no success-batch critic TD (critic_update_sb lax.cond). qc_plus (BC on success actions only): that actor slot runs the critic-only update, actor parameters and optimizer untouched (actor_updates_skipped). No upstream replay-BC fallback"
|
| 741 |
+
},
|
| 742 |
+
"td_mapping": "backbone unified_env_step, gamma_env=0.999 per primitive step, n=5 executed chunks: rewards = backbone reward = sum_{i<L} gamma_env^i r_i, masks = discount / gamma_env^8 with discount = gamma_env^L (1 - done); upstream compute_td_target adds agent.discount^H * masks * Q = gamma_env^L (1 - done) Q; truncation stops the bootstrap; critic_valid=1; valid=executed mask of first chunk",
|
| 743 |
+
"unified_notes": {
|
| 744 |
+
"critic_target_every_critic_step": {
|
| 745 |
+
"note": "matches_upstream: OGPO _critic_target_update and QC target_update run after every critic gradient step (incl. critic-only steps)",
|
| 746 |
+
"value": true
|
| 747 |
+
},
|
| 748 |
+
"success_td_polyak": {
|
| 749 |
+
"note": "matches_upstream: OGPO+ critic_update_sb polyaks the critic target right after the success-batch TD step; QC/QC+ have no success TD step (decision 10)",
|
| 750 |
+
"value": true
|
| 751 |
+
}
|
| 752 |
+
},
|
| 753 |
+
"upstream_config": {
|
| 754 |
+
"agent": {
|
| 755 |
+
"act_bounds_json": "{\"lo\": [-1.0, -1.0, -1.0, -2.9472272396087646, -1.9721344709396362, -1.318867564201355, -1.0], \"hi\": [1.0, 1.0, 1.0, 3.0058202743530273, 1.8893057107925415, 1.0, 1.0]}",
|
| 756 |
+
"act_max": 3.0058202743530273,
|
| 757 |
+
"act_min": -2.9472272396087646,
|
| 758 |
+
"action_chunking": true,
|
| 759 |
+
"action_critic_hidden_dims": null,
|
| 760 |
+
"action_dim": null,
|
| 761 |
+
"action_num_qs": 10,
|
| 762 |
+
"action_repeat": false,
|
| 763 |
+
"actor_backbone": "mlp",
|
| 764 |
+
"actor_decay_steps": 50000,
|
| 765 |
+
"actor_end_value": 2e-05,
|
| 766 |
+
"actor_hidden_dims": [
|
| 767 |
+
512,
|
| 768 |
+
512,
|
| 769 |
+
512,
|
| 770 |
+
512,
|
| 771 |
+
512
|
| 772 |
+
],
|
| 773 |
+
"actor_layer_norm": false,
|
| 774 |
+
"actor_lr": 0.0003,
|
| 775 |
+
"actor_lr_partition": 0.0001,
|
| 776 |
+
"actor_num_samples": 16,
|
| 777 |
+
"actor_obs": "state",
|
| 778 |
+
"actor_scheduler": "cosine",
|
| 779 |
+
"actor_tau": 0.005,
|
| 780 |
+
"actor_type": "best-of-n",
|
| 781 |
+
"actor_warmup_steps": 2000,
|
| 782 |
+
"actor_weight_decay": 0.0,
|
| 783 |
+
"adv_clip_min": null,
|
| 784 |
+
"agent_name": "acfql",
|
| 785 |
+
"alpha": 500,
|
| 786 |
+
"batch_size": 256,
|
| 787 |
+
"best_of_n": 16,
|
| 788 |
+
"bon_agg": "mean",
|
| 789 |
+
"bon_sft": false,
|
| 790 |
+
"bon_sft_coeff": 1.0,
|
| 791 |
+
"bootstrap_ratio": 0.5,
|
| 792 |
+
"bptt_num_samples": 4,
|
| 793 |
+
"calql_q_subsample": 2,
|
| 794 |
+
"clip_grad_norm": 1000.0,
|
| 795 |
+
"clip_intermediate_actions": true,
|
| 796 |
+
"clip_max_epsilon_multiplier": 1.0,
|
| 797 |
+
"clip_min_epsilon_multiplier": 1.0,
|
| 798 |
+
"constant_noise_std": 0.01,
|
| 799 |
+
"cql_alpha": 5.0,
|
| 800 |
+
"cql_n_actions": 10,
|
| 801 |
+
"cql_temp": 1.0,
|
| 802 |
+
"critic_backbone": "mlp",
|
| 803 |
+
"critic_decay_steps": 5000,
|
| 804 |
+
"critic_end_value": 1e-08,
|
| 805 |
+
"critic_loss_type": "mse",
|
| 806 |
+
"critic_lr": 0.0003,
|
| 807 |
+
"critic_lr_partition": 0.0003,
|
| 808 |
+
"critic_obs": "state",
|
| 809 |
+
"critic_scheduler": "constant",
|
| 810 |
+
"critic_warmup_steps": 500,
|
| 811 |
+
"critic_weight_decay": 1e-05,
|
| 812 |
+
"denoised_clip_value": 1.0,
|
| 813 |
+
"discount": 0.999,
|
| 814 |
+
"edit_action_scale": 0.5,
|
| 815 |
+
"edit_actor_hidden_dims": null,
|
| 816 |
+
"edit_hidden_dims": [
|
| 817 |
+
256,
|
| 818 |
+
256,
|
| 819 |
+
256
|
| 820 |
+
],
|
| 821 |
+
"edit_init_temperature": 1.0,
|
| 822 |
+
"edit_layer_norm": true,
|
| 823 |
+
"edit_lr": 0.0003,
|
| 824 |
+
"edit_policy_bound": 0.1,
|
| 825 |
+
"edit_start_step": 100000,
|
| 826 |
+
"edit_target_entropy": null,
|
| 827 |
+
"edit_weight_decay": 0.0,
|
| 828 |
+
"encoder": null,
|
| 829 |
+
"entropy_backup": false,
|
| 830 |
+
"flow_steps": 10,
|
| 831 |
+
"fql_distillation_coeff": 10.0,
|
| 832 |
+
"fql_train_main_policy": true,
|
| 833 |
+
"freeze_vision_encoder": false,
|
| 834 |
+
"ft_flow_steps": 10,
|
| 835 |
+
"horizon_length": 8,
|
| 836 |
+
"init_temperature": 1.0,
|
| 837 |
+
"layer_norm": true,
|
| 838 |
+
"lr": 0.0003,
|
| 839 |
+
"lr_action_critic": null,
|
| 840 |
+
"lr_actor": null,
|
| 841 |
+
"lr_actor_bc_flow": null,
|
| 842 |
+
"lr_critic": null,
|
| 843 |
+
"lr_edit_actor": null,
|
| 844 |
+
"lr_noise_critic": null,
|
| 845 |
+
"max_noise_std": 0.01,
|
| 846 |
+
"mc_regression": false,
|
| 847 |
+
"mc_regression_coeff": 1.0,
|
| 848 |
+
"min_noise_std": 0.01,
|
| 849 |
+
"mip_q_ensemble": false,
|
| 850 |
+
"mip_q_noise_scale": 1.0,
|
| 851 |
+
"mip_q_t_star": 0.9,
|
| 852 |
+
"mip_t_star": 0.999,
|
| 853 |
+
"muon_beta": 0.95,
|
| 854 |
+
"muon_nesterov": true,
|
| 855 |
+
"muon_ns_steps": 5,
|
| 856 |
+
"n_action_samples": 8,
|
| 857 |
+
"n_edit_samples": 8,
|
| 858 |
+
"noise_critic_grad_steps": 1,
|
| 859 |
+
"noise_dim": null,
|
| 860 |
+
"noise_net_hidden_dims": [
|
| 861 |
+
256,
|
| 862 |
+
256
|
| 863 |
+
],
|
| 864 |
+
"noise_net_layer_norm": true,
|
| 865 |
+
"normalize_act_space_dimension": false,
|
| 866 |
+
"normalize_denoising_horizon": false,
|
| 867 |
+
"normalize_q_loss": false,
|
| 868 |
+
"num_bins": 256,
|
| 869 |
+
"num_ensemble_members": 10,
|
| 870 |
+
"num_qs": 10,
|
| 871 |
+
"ob_dims": null,
|
| 872 |
+
"one_step_decay_steps": null,
|
| 873 |
+
"one_step_end_value": null,
|
| 874 |
+
"one_step_hidden_dims": [
|
| 875 |
+
512,
|
| 876 |
+
512,
|
| 877 |
+
512,
|
| 878 |
+
512
|
| 879 |
+
],
|
| 880 |
+
"one_step_layer_norm": false,
|
| 881 |
+
"one_step_lr": null,
|
| 882 |
+
"one_step_scheduler": null,
|
| 883 |
+
"one_step_warmup_steps": null,
|
| 884 |
+
"optimizer_action_critic": null,
|
| 885 |
+
"optimizer_actor": "adam",
|
| 886 |
+
"optimizer_actor_bc_flow": "adam",
|
| 887 |
+
"optimizer_critic": "adam",
|
| 888 |
+
"optimizer_edit_actor": null,
|
| 889 |
+
"paligemma_tokenizer_path": null,
|
| 890 |
+
"partition_optimizer": true,
|
| 891 |
+
"policy_type": "flow",
|
| 892 |
+
"ppo_batch_size": 256,
|
| 893 |
+
"ppo_lr": 4.5e-05,
|
| 894 |
+
"proprio_dim": null,
|
| 895 |
+
"q_agg": "mean",
|
| 896 |
+
"q_max": null,
|
| 897 |
+
"q_min": null,
|
| 898 |
+
"q_target_agg": "subset_min",
|
| 899 |
+
"q_target_replace": false,
|
| 900 |
+
"q_target_subset": 2,
|
| 901 |
+
"q_variance_reduction": false,
|
| 902 |
+
"q_vr_num_samples": 8,
|
| 903 |
+
"q_vr_reduction": "mean",
|
| 904 |
+
"randn_clip_value": 3.0,
|
| 905 |
+
"shortcut_denoising_steps": 4,
|
| 906 |
+
"shortcut_inference_steps": 4,
|
| 907 |
+
"simba_actor_hidden_dims": [
|
| 908 |
+
1024,
|
| 909 |
+
1024
|
| 910 |
+
],
|
| 911 |
+
"simba_critic_hidden_dims": [
|
| 912 |
+
1024,
|
| 913 |
+
1024,
|
| 914 |
+
1024
|
| 915 |
+
],
|
| 916 |
+
"subsample_bon": false,
|
| 917 |
+
"target_entropy": null,
|
| 918 |
+
"tau": 0.005,
|
| 919 |
+
"tf_pi_conditioning": "adaln",
|
| 920 |
+
"tf_pi_dropout": 0.0,
|
| 921 |
+
"tf_pi_embed_dim": 256,
|
| 922 |
+
"tf_pi_heads": 4,
|
| 923 |
+
"tf_pi_layers": 8,
|
| 924 |
+
"tf_q_conditioning": "adaln",
|
| 925 |
+
"tf_q_dropout": 0.0,
|
| 926 |
+
"tf_q_embed_dim": 256,
|
| 927 |
+
"tf_q_heads": 4,
|
| 928 |
+
"tf_q_layers": 8,
|
| 929 |
+
"time_embedding": "sinusoidal",
|
| 930 |
+
"time_embedding_dim": 32,
|
| 931 |
+
"train_one_step_in_offline": true,
|
| 932 |
+
"train_one_step_in_online": true,
|
| 933 |
+
"useFiLM": false,
|
| 934 |
+
"useSimBa": false,
|
| 935 |
+
"useTF": false,
|
| 936 |
+
"use_constant_noise": true,
|
| 937 |
+
"use_constant_scheduler_for_bc": true,
|
| 938 |
+
"use_denoiser": false,
|
| 939 |
+
"use_edit_actor": false,
|
| 940 |
+
"use_mip_q": false,
|
| 941 |
+
"use_muon": false,
|
| 942 |
+
"use_one_step_policy": false,
|
| 943 |
+
"use_shortcut": false,
|
| 944 |
+
"use_state": "proprio",
|
| 945 |
+
"use_success_buffer_q": false,
|
| 946 |
+
"value_hidden_dims": [
|
| 947 |
+
512,
|
| 948 |
+
512,
|
| 949 |
+
512
|
| 950 |
+
],
|
| 951 |
+
"vit_encoder": "minvit",
|
| 952 |
+
"vit_img_proj_dim": 0,
|
| 953 |
+
"vit_state_proj_dim": 0
|
| 954 |
+
},
|
| 955 |
+
"bc": {
|
| 956 |
+
"clip_bc": true,
|
| 957 |
+
"clip_bc_threshold": 0.45,
|
| 958 |
+
"clip_bc_wrt": "ODE"
|
| 959 |
+
},
|
| 960 |
+
"buffer": {
|
| 961 |
+
"offline_ratio": 0.0,
|
| 962 |
+
"size": 2000000,
|
| 963 |
+
"start_training": 10000,
|
| 964 |
+
"utd_online": 1,
|
| 965 |
+
"utd_pi": null,
|
| 966 |
+
"utd_q": null,
|
| 967 |
+
"utd_v": null,
|
| 968 |
+
"utd_warmup": 1
|
| 969 |
+
},
|
| 970 |
+
"checkpoint": {
|
| 971 |
+
"ep_resume": 0,
|
| 972 |
+
"restore_actor_path": null,
|
| 973 |
+
"restore_critic_path": null,
|
| 974 |
+
"restore_encoder_path": null
|
| 975 |
+
},
|
| 976 |
+
"dataset": {
|
| 977 |
+
"num_offline_trajs": -1,
|
| 978 |
+
"ogbench_dir": null,
|
| 979 |
+
"p_aug": 0.0,
|
| 980 |
+
"proportion": 1.0,
|
| 981 |
+
"replace_interval": 1000
|
| 982 |
+
},
|
| 983 |
+
"env": {
|
| 984 |
+
"name": "tool_hang-ph-low_dim",
|
| 985 |
+
"post_success_steps": 0,
|
| 986 |
+
"rew_fn": "sparse",
|
| 987 |
+
"sparse": false
|
| 988 |
+
},
|
| 989 |
+
"eval": {
|
| 990 |
+
"eval_episodes": 2,
|
| 991 |
+
"n_eval_envs": 32,
|
| 992 |
+
"plot_q_vs_mc": false,
|
| 993 |
+
"video_episodes": 0,
|
| 994 |
+
"video_frame_skip": 3
|
| 995 |
+
},
|
| 996 |
+
"experiment": {
|
| 997 |
+
"log": true,
|
| 998 |
+
"project": "QC",
|
| 999 |
+
"run_group": "toolhang",
|
| 1000 |
+
"run_name": "qc_toolhang",
|
| 1001 |
+
"save_dir": "exp/ogpo",
|
| 1002 |
+
"seed": 1,
|
| 1003 |
+
"wandb_name": "qc_toolhang"
|
| 1004 |
+
},
|
| 1005 |
+
"intervals": {
|
| 1006 |
+
"eval_interval": 20000,
|
| 1007 |
+
"eval_interval_bc": 5000,
|
| 1008 |
+
"log_interval": 5000,
|
| 1009 |
+
"preemption_backup_interval": -1,
|
| 1010 |
+
"save_interval": 500000,
|
| 1011 |
+
"save_interval_bc": -1,
|
| 1012 |
+
"step_data_log_interval": -1
|
| 1013 |
+
},
|
| 1014 |
+
"success_buffer": {
|
| 1015 |
+
"batch_size": 256,
|
| 1016 |
+
"use_success_buffer": false
|
| 1017 |
+
},
|
| 1018 |
+
"training": {
|
| 1019 |
+
"bc_only": false,
|
| 1020 |
+
"bc_pi_steps": 0,
|
| 1021 |
+
"bc_q_steps": 0,
|
| 1022 |
+
"bc_refine_steps": 0,
|
| 1023 |
+
"calql_steps": 0,
|
| 1024 |
+
"force_jax_sync": true,
|
| 1025 |
+
"gpu_mode": "single",
|
| 1026 |
+
"online_steps": 5000000,
|
| 1027 |
+
"q_warmup_steps": 0
|
| 1028 |
+
}
|
| 1029 |
+
},
|
| 1030 |
+
"upstream_source": {
|
| 1031 |
+
"ogpo_commit": "bfe2eff74ede15121552256a7d76f02ba840a63d",
|
| 1032 |
+
"patches_sha256": {
|
| 1033 |
+
"ogpo-pixel-backbone.patch": "cac13396730addcfbb5cd4c80e9727fbec08f0336ff2e51acc9fbde23f5322a5",
|
| 1034 |
+
"ogpo-qc-cfm-base-architecture.patch": "d7f2077b282d475a51d9fb0135860523f02ad59f9e821cd54c24fb534ee959f5",
|
| 1035 |
+
"ogpo-runner-storage-hooks.patch": "255f21212530023114729b037ed448b3ecaee05693548163de5cf7a91389ff26",
|
| 1036 |
+
"ogpo-study2-0930-decisions.patch": "fb286cee046b846c4301a69716086e1c0d3ee0e0de6fc6b111bdf7647b239bc2",
|
| 1037 |
+
"ogpo-study2-critic-ensemble.patch": "cab15db410a027f6a6bf58292f70331aa19bc39f0f156ea542d355662795e319"
|
| 1038 |
+
}
|
| 1039 |
+
}
|
| 1040 |
+
},
|
| 1041 |
+
"observation_mode": "pixel_state",
|
| 1042 |
+
"offline_buffer": {
|
| 1043 |
+
"actions_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1044 |
+
"aux_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 1045 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 1046 |
+
"chunks": 6171,
|
| 1047 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1048 |
+
"decision_ref": "[1017-S2OFFBC]",
|
| 1049 |
+
"env_steps": 48984,
|
| 1050 |
+
"features_sha256": "fb059cee8121cd0039a4ac218f1a19de767b07cf81010471e4342480fc9119fa",
|
| 1051 |
+
"horizon": 8,
|
| 1052 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 1053 |
+
"learner_coordinates": "JaxStudy2Learner:qc",
|
| 1054 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1055 |
+
"ratio": {
|
| 1056 |
+
"adaptive_ratio": false,
|
| 1057 |
+
"rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
|
| 1058 |
+
},
|
| 1059 |
+
"selection": {
|
| 1060 |
+
"cohort_size": 100,
|
| 1061 |
+
"demos": [
|
| 1062 |
+
"demo_0",
|
| 1063 |
+
"demo_2",
|
| 1064 |
+
"demo_5",
|
| 1065 |
+
"demo_7",
|
| 1066 |
+
"demo_9",
|
| 1067 |
+
"demo_10",
|
| 1068 |
+
"demo_13",
|
| 1069 |
+
"demo_16",
|
| 1070 |
+
"demo_17",
|
| 1071 |
+
"demo_18",
|
| 1072 |
+
"demo_20",
|
| 1073 |
+
"demo_23",
|
| 1074 |
+
"demo_27",
|
| 1075 |
+
"demo_28",
|
| 1076 |
+
"demo_29",
|
| 1077 |
+
"demo_30",
|
| 1078 |
+
"demo_31",
|
| 1079 |
+
"demo_32",
|
| 1080 |
+
"demo_34",
|
| 1081 |
+
"demo_35",
|
| 1082 |
+
"demo_37",
|
| 1083 |
+
"demo_40",
|
| 1084 |
+
"demo_41",
|
| 1085 |
+
"demo_43",
|
| 1086 |
+
"demo_46",
|
| 1087 |
+
"demo_47",
|
| 1088 |
+
"demo_50",
|
| 1089 |
+
"demo_51",
|
| 1090 |
+
"demo_56",
|
| 1091 |
+
"demo_57",
|
| 1092 |
+
"demo_58",
|
| 1093 |
+
"demo_67",
|
| 1094 |
+
"demo_71",
|
| 1095 |
+
"demo_74",
|
| 1096 |
+
"demo_76",
|
| 1097 |
+
"demo_78",
|
| 1098 |
+
"demo_80",
|
| 1099 |
+
"demo_82",
|
| 1100 |
+
"demo_83",
|
| 1101 |
+
"demo_85",
|
| 1102 |
+
"demo_86",
|
| 1103 |
+
"demo_87",
|
| 1104 |
+
"demo_93",
|
| 1105 |
+
"demo_97",
|
| 1106 |
+
"demo_98",
|
| 1107 |
+
"demo_101",
|
| 1108 |
+
"demo_102",
|
| 1109 |
+
"demo_107",
|
| 1110 |
+
"demo_110",
|
| 1111 |
+
"demo_111",
|
| 1112 |
+
"demo_112",
|
| 1113 |
+
"demo_113",
|
| 1114 |
+
"demo_115",
|
| 1115 |
+
"demo_116",
|
| 1116 |
+
"demo_119",
|
| 1117 |
+
"demo_122",
|
| 1118 |
+
"demo_123",
|
| 1119 |
+
"demo_126",
|
| 1120 |
+
"demo_127",
|
| 1121 |
+
"demo_128",
|
| 1122 |
+
"demo_130",
|
| 1123 |
+
"demo_131",
|
| 1124 |
+
"demo_132",
|
| 1125 |
+
"demo_133",
|
| 1126 |
+
"demo_134",
|
| 1127 |
+
"demo_135",
|
| 1128 |
+
"demo_138",
|
| 1129 |
+
"demo_140",
|
| 1130 |
+
"demo_141",
|
| 1131 |
+
"demo_145",
|
| 1132 |
+
"demo_147",
|
| 1133 |
+
"demo_149",
|
| 1134 |
+
"demo_151",
|
| 1135 |
+
"demo_152",
|
| 1136 |
+
"demo_154",
|
| 1137 |
+
"demo_156",
|
| 1138 |
+
"demo_157",
|
| 1139 |
+
"demo_159",
|
| 1140 |
+
"demo_161",
|
| 1141 |
+
"demo_163",
|
| 1142 |
+
"demo_164",
|
| 1143 |
+
"demo_171",
|
| 1144 |
+
"demo_172",
|
| 1145 |
+
"demo_173",
|
| 1146 |
+
"demo_174",
|
| 1147 |
+
"demo_175",
|
| 1148 |
+
"demo_176",
|
| 1149 |
+
"demo_177",
|
| 1150 |
+
"demo_179",
|
| 1151 |
+
"demo_181",
|
| 1152 |
+
"demo_182",
|
| 1153 |
+
"demo_183",
|
| 1154 |
+
"demo_186",
|
| 1155 |
+
"demo_188",
|
| 1156 |
+
"demo_190",
|
| 1157 |
+
"demo_191",
|
| 1158 |
+
"demo_192",
|
| 1159 |
+
"demo_193",
|
| 1160 |
+
"demo_196",
|
| 1161 |
+
"demo_199"
|
| 1162 |
+
],
|
| 1163 |
+
"manifest": "/ssd/WorkSpace/RL-Postrain/checkpoints/huggingface/yangfy0627/RL-postTrain/e62671ed1cf646b453660b698c22ca4d273b5ccc/Robomimic/study1/ToolHang/diverse100/seed_1/data_manifest.json",
|
| 1164 |
+
"manifest_sha256": "9233d5824205e29f1afa721d8573fe19805a1a8e7829c296af1dd0b74a1d29b4",
|
| 1165 |
+
"regime": "diverse100",
|
| 1166 |
+
"source": "bc_regime",
|
| 1167 |
+
"source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 1168 |
+
"split": {
|
| 1169 |
+
"training": 90,
|
| 1170 |
+
"validation": 10
|
| 1171 |
+
},
|
| 1172 |
+
"task": "tool_hang"
|
| 1173 |
+
}
|
| 1174 |
+
},
|
| 1175 |
+
"offline_role": "mixed",
|
| 1176 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 1177 |
+
"smoke": false,
|
| 1178 |
+
"task": "tool_hang",
|
| 1179 |
+
"task_spec_hash": "ac2909345e9536690a9da63f06aad8dfe135e757b89e28706b2c4af641f195c2"
|
| 1180 |
+
},
|
| 1181 |
+
"seed": 1,
|
| 1182 |
+
"start_policy": "base_cfm_ode",
|
| 1183 |
+
"variant": {
|
| 1184 |
+
"flags": {
|
| 1185 |
+
"bon_sft": false,
|
| 1186 |
+
"use_success_buffer": false
|
| 1187 |
+
},
|
| 1188 |
+
"learner": "qc",
|
| 1189 |
+
"name": "qc",
|
| 1190 |
+
"phase": 1,
|
| 1191 |
+
"spec_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
|
| 1192 |
+
"version": "study2_variants_v4"
|
| 1193 |
+
},
|
| 1194 |
+
"variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2"
|
| 1195 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/start_protocol.json
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"committed_episodes": [],
|
| 3 |
+
"mode": "offline_buffer",
|
| 4 |
+
"note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
|
| 5 |
+
"offline": {
|
| 6 |
+
"adaptive_ratio": false,
|
| 7 |
+
"batches": 0,
|
| 8 |
+
"decay_env_steps": null,
|
| 9 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 10 |
+
"last_ratio": null,
|
| 11 |
+
"offline_rows": 0,
|
| 12 |
+
"online_rows": 0,
|
| 13 |
+
"ratio_end": null,
|
| 14 |
+
"ratio_start": null,
|
| 15 |
+
"records": 6171
|
| 16 |
+
},
|
| 17 |
+
"offline_identity": {
|
| 18 |
+
"actions_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 19 |
+
"aux_sha256": "7d65f5b7df78085d4780cf6338ac032e63f7f3d6ff9b1923bcc0992c840ee2d9",
|
| 20 |
+
"checkpoint_sha256": "c5381a3e2ff4f4eeeb899a9d14286d9d42e0816c6431f1208fa39c60309c48f8",
|
| 21 |
+
"chunks": 6171,
|
| 22 |
+
"dataset_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 23 |
+
"decision_ref": "[1017-S2OFFBC]",
|
| 24 |
+
"env_steps": 48984,
|
| 25 |
+
"features_sha256": "fb059cee8121cd0039a4ac218f1a19de767b07cf81010471e4342480fc9119fa",
|
| 26 |
+
"horizon": 8,
|
| 27 |
+
"learner_coordinates": "JaxStudy2Learner:qc",
|
| 28 |
+
"pixel_spec_hash": "7aba3118e0304ea7265915550734918674f2b9a1e6c9bc2eb1cd99b568045a11",
|
| 29 |
+
"ratio": {
|
| 30 |
+
"adaptive_ratio": false,
|
| 31 |
+
"rule": "D_demo kept for the whole run (never evicted), online RL data FIFO; each ordinary batch = batch_size uniform draws over the eligible D_demo + online windows, offline rows first"
|
| 32 |
+
},
|
| 33 |
+
"selection": {
|
| 34 |
+
"cohort_size": 100,
|
| 35 |
+
"demos": [
|
| 36 |
+
"demo_0",
|
| 37 |
+
"demo_2",
|
| 38 |
+
"demo_5",
|
| 39 |
+
"demo_7",
|
| 40 |
+
"demo_9",
|
| 41 |
+
"demo_10",
|
| 42 |
+
"demo_13",
|
| 43 |
+
"demo_16",
|
| 44 |
+
"demo_17",
|
| 45 |
+
"demo_18",
|
| 46 |
+
"demo_20",
|
| 47 |
+
"demo_23",
|
| 48 |
+
"demo_27",
|
| 49 |
+
"demo_28",
|
| 50 |
+
"demo_29",
|
| 51 |
+
"demo_30",
|
| 52 |
+
"demo_31",
|
| 53 |
+
"demo_32",
|
| 54 |
+
"demo_34",
|
| 55 |
+
"demo_35",
|
| 56 |
+
"demo_37",
|
| 57 |
+
"demo_40",
|
| 58 |
+
"demo_41",
|
| 59 |
+
"demo_43",
|
| 60 |
+
"demo_46",
|
| 61 |
+
"demo_47",
|
| 62 |
+
"demo_50",
|
| 63 |
+
"demo_51",
|
| 64 |
+
"demo_56",
|
| 65 |
+
"demo_57",
|
| 66 |
+
"demo_58",
|
| 67 |
+
"demo_67",
|
| 68 |
+
"demo_71",
|
| 69 |
+
"demo_74",
|
| 70 |
+
"demo_76",
|
| 71 |
+
"demo_78",
|
| 72 |
+
"demo_80",
|
| 73 |
+
"demo_82",
|
| 74 |
+
"demo_83",
|
| 75 |
+
"demo_85",
|
| 76 |
+
"demo_86",
|
| 77 |
+
"demo_87",
|
| 78 |
+
"demo_93",
|
| 79 |
+
"demo_97",
|
| 80 |
+
"demo_98",
|
| 81 |
+
"demo_101",
|
| 82 |
+
"demo_102",
|
| 83 |
+
"demo_107",
|
| 84 |
+
"demo_110",
|
| 85 |
+
"demo_111",
|
| 86 |
+
"demo_112",
|
| 87 |
+
"demo_113",
|
| 88 |
+
"demo_115",
|
| 89 |
+
"demo_116",
|
| 90 |
+
"demo_119",
|
| 91 |
+
"demo_122",
|
| 92 |
+
"demo_123",
|
| 93 |
+
"demo_126",
|
| 94 |
+
"demo_127",
|
| 95 |
+
"demo_128",
|
| 96 |
+
"demo_130",
|
| 97 |
+
"demo_131",
|
| 98 |
+
"demo_132",
|
| 99 |
+
"demo_133",
|
| 100 |
+
"demo_134",
|
| 101 |
+
"demo_135",
|
| 102 |
+
"demo_138",
|
| 103 |
+
"demo_140",
|
| 104 |
+
"demo_141",
|
| 105 |
+
"demo_145",
|
| 106 |
+
"demo_147",
|
| 107 |
+
"demo_149",
|
| 108 |
+
"demo_151",
|
| 109 |
+
"demo_152",
|
| 110 |
+
"demo_154",
|
| 111 |
+
"demo_156",
|
| 112 |
+
"demo_157",
|
| 113 |
+
"demo_159",
|
| 114 |
+
"demo_161",
|
| 115 |
+
"demo_163",
|
| 116 |
+
"demo_164",
|
| 117 |
+
"demo_171",
|
| 118 |
+
"demo_172",
|
| 119 |
+
"demo_173",
|
| 120 |
+
"demo_174",
|
| 121 |
+
"demo_175",
|
| 122 |
+
"demo_176",
|
| 123 |
+
"demo_177",
|
| 124 |
+
"demo_179",
|
| 125 |
+
"demo_181",
|
| 126 |
+
"demo_182",
|
| 127 |
+
"demo_183",
|
| 128 |
+
"demo_186",
|
| 129 |
+
"demo_188",
|
| 130 |
+
"demo_190",
|
| 131 |
+
"demo_191",
|
| 132 |
+
"demo_192",
|
| 133 |
+
"demo_193",
|
| 134 |
+
"demo_196",
|
| 135 |
+
"demo_199"
|
| 136 |
+
],
|
| 137 |
+
"manifest": "/ssd/WorkSpace/RL-Postrain/checkpoints/huggingface/yangfy0627/RL-postTrain/e62671ed1cf646b453660b698c22ca4d273b5ccc/Robomimic/study1/ToolHang/diverse100/seed_1/data_manifest.json",
|
| 138 |
+
"manifest_sha256": "9233d5824205e29f1afa721d8573fe19805a1a8e7829c296af1dd0b74a1d29b4",
|
| 139 |
+
"regime": "diverse100",
|
| 140 |
+
"source": "bc_regime",
|
| 141 |
+
"source_ph_sha256": "c5cb01b119a109a3d4842ca515906e86f1f35a8ee1a333d22b3503c1a513a8f4",
|
| 142 |
+
"split": {
|
| 143 |
+
"training": 90,
|
| 144 |
+
"validation": 10
|
| 145 |
+
},
|
| 146 |
+
"task": "tool_hang"
|
| 147 |
+
}
|
| 148 |
+
},
|
| 149 |
+
"policy": "base_cfm_ode",
|
| 150 |
+
"start_chunks": 0,
|
| 151 |
+
"start_env_steps": 0,
|
| 152 |
+
"status": "complete"
|
| 153 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/status.json
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"accounting": {
|
| 3 |
+
"evaluation": {
|
| 4 |
+
"calls": 82,
|
| 5 |
+
"episodes": 16500
|
| 6 |
+
},
|
| 7 |
+
"history_complete": true,
|
| 8 |
+
"normalized": {
|
| 9 |
+
"checkpoint_seconds_per_env_step": 4.571324955062282e-05,
|
| 10 |
+
"chunks_per_10000_env_steps": 1251.2652719727785,
|
| 11 |
+
"denominator_env_steps": 1606374,
|
| 12 |
+
"diagnostics_seconds_per_env_step": 0.00010274175891293023,
|
| 13 |
+
"episodes_per_10000_env_steps": 11.280063048829227,
|
| 14 |
+
"evaluation_seconds_per_env_step": 0.05122841758099049,
|
| 15 |
+
"history_complete": true,
|
| 16 |
+
"projected_chunks": 200977.7274781589,
|
| 17 |
+
"projected_episodes": 1811.799214877731,
|
| 18 |
+
"projected_training_seconds": 28151.4625361975,
|
| 19 |
+
"projected_wall_seconds": 110730.6731631028,
|
| 20 |
+
"projection_budget_env_steps": 1606196,
|
| 21 |
+
"projection_is_estimate": true,
|
| 22 |
+
"rollout_seconds_per_env_step": 0.005669967505914039,
|
| 23 |
+
"training_seconds_per_env_step": 0.017526791584711644,
|
| 24 |
+
"updates_seconds_per_env_step": 0.011856824078797604,
|
| 25 |
+
"wall_seconds_per_env_step": 0.06893970173198215
|
| 26 |
+
},
|
| 27 |
+
"rollout": {
|
| 28 |
+
"chunks": 201000,
|
| 29 |
+
"env_steps": 1606374,
|
| 30 |
+
"episodes": 1812,
|
| 31 |
+
"start_chunks": 0,
|
| 32 |
+
"start_env_steps": 0,
|
| 33 |
+
"successes": 447,
|
| 34 |
+
"truncated_episodes": 1365
|
| 35 |
+
},
|
| 36 |
+
"schema_version": 1,
|
| 37 |
+
"sequence": 1375,
|
| 38 |
+
"sessions": 1,
|
| 39 |
+
"time": {
|
| 40 |
+
"checkpoint_seconds": 73.43257553363219,
|
| 41 |
+
"diagnostics_seconds": 165.04169023199938,
|
| 42 |
+
"evaluation_seconds": 82291.99806324602,
|
| 43 |
+
"preparation_seconds": 53.02641303115524,
|
| 44 |
+
"restore_seconds": 0.0,
|
| 45 |
+
"rollout_seconds": 9108.088382345159,
|
| 46 |
+
"start_rollout_seconds": 0.0,
|
| 47 |
+
"training_seconds": 28154.582305099582,
|
| 48 |
+
"updates_seconds": 19046.493922754424,
|
| 49 |
+
"wall_hours": 30.761929008336413,
|
| 50 |
+
"wall_seconds": 110742.94443001109
|
| 51 |
+
}
|
| 52 |
+
},
|
| 53 |
+
"detail": null,
|
| 54 |
+
"evals": 81,
|
| 55 |
+
"final_eval": {
|
| 56 |
+
"episodes": 300,
|
| 57 |
+
"imputed": false,
|
| 58 |
+
"n_success": 77,
|
| 59 |
+
"observed": true,
|
| 60 |
+
"seed_start": 800000,
|
| 61 |
+
"sr": 0.25666666666666665
|
| 62 |
+
},
|
| 63 |
+
"learner": "qc",
|
| 64 |
+
"learner_counters": {
|
| 65 |
+
"actor_updates": 301500,
|
| 66 |
+
"actor_updates_applied": 301500,
|
| 67 |
+
"actor_updates_skipped": 0,
|
| 68 |
+
"clip_stats": {
|
| 69 |
+
"clip_mag_sum": 1703.4173249006271,
|
| 70 |
+
"coords_clipped_frac": 0.0293637171286425,
|
| 71 |
+
"max_clip_mag": 0.35409653186798096,
|
| 72 |
+
"mean_clip_mag": 0.005153780807401192,
|
| 73 |
+
"n_clipped": 330518,
|
| 74 |
+
"n_coords": 11256000
|
| 75 |
+
},
|
| 76 |
+
"clip_stats_start": {
|
| 77 |
+
"clip_mag_sum": 0.0,
|
| 78 |
+
"coords_clipped_frac": 0.0,
|
| 79 |
+
"max_clip_mag": 0.0,
|
| 80 |
+
"mean_clip_mag": 0.0,
|
| 81 |
+
"n_clipped": 0,
|
| 82 |
+
"n_coords": 0
|
| 83 |
+
},
|
| 84 |
+
"critic_updates": 603000,
|
| 85 |
+
"policy_version": 201,
|
| 86 |
+
"success_missing": 0
|
| 87 |
+
},
|
| 88 |
+
"phase": 1,
|
| 89 |
+
"protocol_hash": "903b284201350b7c879df61bb6860ec2d7fc9e4537fb5abd2aba8bcf7a60ee0f",
|
| 90 |
+
"resumes": [],
|
| 91 |
+
"schedule": {
|
| 92 |
+
"actor_updates": 301500,
|
| 93 |
+
"chunk_transitions": 201000,
|
| 94 |
+
"critic_updates": 603000,
|
| 95 |
+
"gradient_updates": 603000,
|
| 96 |
+
"rl_chunks": 201000,
|
| 97 |
+
"rl_env_steps": 1606374,
|
| 98 |
+
"round_index": 201,
|
| 99 |
+
"update_index": 603000
|
| 100 |
+
},
|
| 101 |
+
"seed": 1,
|
| 102 |
+
"start": {
|
| 103 |
+
"mode": "offline_buffer",
|
| 104 |
+
"note": "[1006-S2V5] no BC start rollout; RL collects from step 0",
|
| 105 |
+
"offline": {
|
| 106 |
+
"adaptive_ratio": false,
|
| 107 |
+
"batches": 603000,
|
| 108 |
+
"decay_env_steps": null,
|
| 109 |
+
"identity_hash": "8c64afb29981d413f8039166e0107b36908bfa33068ff594d65b9275a0beb1db",
|
| 110 |
+
"last_ratio": 0.029789863432954703,
|
| 111 |
+
"offline_rows": 16298500,
|
| 112 |
+
"online_rows": 138069500,
|
| 113 |
+
"ratio_end": null,
|
| 114 |
+
"ratio_start": null,
|
| 115 |
+
"records": 6171
|
| 116 |
+
},
|
| 117 |
+
"start_chunks": 0,
|
| 118 |
+
"start_env_steps": 0,
|
| 119 |
+
"status": "complete"
|
| 120 |
+
},
|
| 121 |
+
"status": "budget_complete",
|
| 122 |
+
"stop": {
|
| 123 |
+
"eval_index": 80,
|
| 124 |
+
"gain": -0.00599999999999995,
|
| 125 |
+
"ma": 0.23000000000000004,
|
| 126 |
+
"rl_env_steps": 1606374,
|
| 127 |
+
"sr": 0.22,
|
| 128 |
+
"status": "budget_complete",
|
| 129 |
+
"trend": "flat"
|
| 130 |
+
},
|
| 131 |
+
"success_sampling": {
|
| 132 |
+
"imitation": {
|
| 133 |
+
"requested": 0,
|
| 134 |
+
"rounds_with_skips": 0,
|
| 135 |
+
"sampled": 0,
|
| 136 |
+
"skipped": 0
|
| 137 |
+
},
|
| 138 |
+
"last_round": {
|
| 139 |
+
"imitation": {
|
| 140 |
+
"requested": 0,
|
| 141 |
+
"sampled": 0,
|
| 142 |
+
"skipped": 0
|
| 143 |
+
},
|
| 144 |
+
"requested": 0,
|
| 145 |
+
"round": 200,
|
| 146 |
+
"sampled": 0,
|
| 147 |
+
"skipped": 0
|
| 148 |
+
},
|
| 149 |
+
"requested": 0,
|
| 150 |
+
"rounds_with_skips": 0,
|
| 151 |
+
"sampled": 0,
|
| 152 |
+
"skipped": 0,
|
| 153 |
+
"td": {
|
| 154 |
+
"requested": 0,
|
| 155 |
+
"rounds_with_skips": 0,
|
| 156 |
+
"sampled": 0,
|
| 157 |
+
"skipped": 0
|
| 158 |
+
}
|
| 159 |
+
},
|
| 160 |
+
"variant": "qc",
|
| 161 |
+
"variant_hash": "04a691a48833a81c27289de26e928af6d41f006b496bb7d391e8a569d7b428d2",
|
| 162 |
+
"wandb": {
|
| 163 |
+
"enabled": false,
|
| 164 |
+
"error": null,
|
| 165 |
+
"finished": true,
|
| 166 |
+
"mode": "online",
|
| 167 |
+
"name": "ToolHang-diverse100-QC",
|
| 168 |
+
"project": "RL-postTrain",
|
| 169 |
+
"reason": null,
|
| 170 |
+
"run_id": "23871f6e",
|
| 171 |
+
"transport": "relay",
|
| 172 |
+
"url": "https://wandb.ai/f70331658-university-of-chinese-academy-of-sciences/RL-postTrain/runs/23871f6e"
|
| 173 |
+
}
|
| 174 |
+
}
|
RL/ToolHang100/QC/diverse100/seed_1/training_accounting.json
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"evaluation": {
|
| 3 |
+
"calls": 82,
|
| 4 |
+
"episodes": 16500
|
| 5 |
+
},
|
| 6 |
+
"history_complete": true,
|
| 7 |
+
"normalized": {
|
| 8 |
+
"checkpoint_seconds_per_env_step": 4.571324955062282e-05,
|
| 9 |
+
"chunks_per_10000_env_steps": 1251.2652719727785,
|
| 10 |
+
"denominator_env_steps": 1606374,
|
| 11 |
+
"diagnostics_seconds_per_env_step": 0.00010274175891293023,
|
| 12 |
+
"episodes_per_10000_env_steps": 11.280063048829227,
|
| 13 |
+
"evaluation_seconds_per_env_step": 0.05122841758099049,
|
| 14 |
+
"history_complete": true,
|
| 15 |
+
"projected_chunks": 200977.7274781589,
|
| 16 |
+
"projected_episodes": 1811.799214877731,
|
| 17 |
+
"projected_training_seconds": 28151.4625361975,
|
| 18 |
+
"projected_wall_seconds": 110730.6731631028,
|
| 19 |
+
"projection_budget_env_steps": 1606196,
|
| 20 |
+
"projection_is_estimate": true,
|
| 21 |
+
"rollout_seconds_per_env_step": 0.005669967505914039,
|
| 22 |
+
"training_seconds_per_env_step": 0.017526791584711644,
|
| 23 |
+
"updates_seconds_per_env_step": 0.011856824078797604,
|
| 24 |
+
"wall_seconds_per_env_step": 0.06893970173198215
|
| 25 |
+
},
|
| 26 |
+
"rollout": {
|
| 27 |
+
"chunks": 201000,
|
| 28 |
+
"env_steps": 1606374,
|
| 29 |
+
"episodes": 1812,
|
| 30 |
+
"start_chunks": 0,
|
| 31 |
+
"start_env_steps": 0,
|
| 32 |
+
"successes": 447,
|
| 33 |
+
"truncated_episodes": 1365
|
| 34 |
+
},
|
| 35 |
+
"schema_version": 1,
|
| 36 |
+
"sequence": 1376,
|
| 37 |
+
"sessions": 1,
|
| 38 |
+
"time": {
|
| 39 |
+
"checkpoint_seconds": 73.43257553363219,
|
| 40 |
+
"diagnostics_seconds": 165.04169023199938,
|
| 41 |
+
"evaluation_seconds": 82291.99806324602,
|
| 42 |
+
"preparation_seconds": 53.02641303115524,
|
| 43 |
+
"restore_seconds": 0.0,
|
| 44 |
+
"rollout_seconds": 9108.088382345159,
|
| 45 |
+
"start_rollout_seconds": 0.0,
|
| 46 |
+
"training_seconds": 28154.582305099582,
|
| 47 |
+
"updates_seconds": 19046.493922754424,
|
| 48 |
+
"wall_hours": 30.761929008336413,
|
| 49 |
+
"wall_seconds": 110742.94443001109
|
| 50 |
+
}
|
| 51 |
+
}
|