train smoke progress
Browse files- ft1/ft_progress.json +69 -1
ft1/ft_progress.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"stage": "
|
| 3 |
"t0": 1790629770.2236812,
|
| 4 |
"events": [
|
| 5 |
[
|
|
@@ -33,6 +33,74 @@
|
|
| 33 |
[
|
| 34 |
288.5,
|
| 35 |
"cloning isaaclab v2.3.0"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
]
|
| 37 |
]
|
| 38 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"stage": "stageA7",
|
| 3 |
"t0": 1790629770.2236812,
|
| 4 |
"events": [
|
| 5 |
[
|
|
|
|
| 33 |
[
|
| 34 |
288.5,
|
| 35 |
"cloning isaaclab v2.3.0"
|
| 36 |
+
],
|
| 37 |
+
[
|
| 38 |
+
301.8,
|
| 39 |
+
"cloning GR00T-WholeBodyControl (with lfs)"
|
| 40 |
+
],
|
| 41 |
+
[
|
| 42 |
+
354.9,
|
| 43 |
+
"downloading junsooki checkpoint + sample motions"
|
| 44 |
+
],
|
| 45 |
+
[
|
| 46 |
+
361.7,
|
| 47 |
+
"RESUME ckpt ft1/h2_medv_ft_last.pt (v36 final) - NO surgery (obs 1114 + std 0.15 already in ckpt)"
|
| 48 |
+
],
|
| 49 |
+
[
|
| 50 |
+
405.0,
|
| 51 |
+
"WARM ckpt stageA6_last -> /tmp/resume_a7/init.pt (v2i: +resume=true restores adp failure stats + optimizer + env state; trainer state STRIPPED - restored global_step>=max_steps caused early stop in v2h)"
|
| 52 |
+
],
|
| 53 |
+
[
|
| 54 |
+
405.1,
|
| 55 |
+
"v30: MEDV-ONLY motion set (no sample pkls) - 100% rally sampling"
|
| 56 |
+
],
|
| 57 |
+
[
|
| 58 |
+
405.6,
|
| 59 |
+
"medv_rally1 motionlib added (key-aligned name)"
|
| 60 |
+
],
|
| 61 |
+
[
|
| 62 |
+
406.2,
|
| 63 |
+
"real smpl file added (medv 50fps flat72)"
|
| 64 |
+
],
|
| 65 |
+
[
|
| 66 |
+
406.2,
|
| 67 |
+
"check_environment.py --training"
|
| 68 |
+
],
|
| 69 |
+
[
|
| 70 |
+
411.7,
|
| 71 |
+
"IRAY LDD: ls: cannot access '/kaggle/working/v311/lib/python3.11/site-packages/isaacsim/extscache/omni.iray.lib': No such file or directory"
|
| 72 |
+
],
|
| 73 |
+
[
|
| 74 |
+
411.7,
|
| 75 |
+
"launching 10-iter smoke train"
|
| 76 |
+
],
|
| 77 |
+
[
|
| 78 |
+
411.8,
|
| 79 |
+
"v2f: brake pre-step SKIPPED - 0.30 probe PASSED in v2e (max anchor err 0.269m < 0.30 radius, 800 samples, ref-act applied 400/400; segments f0-3 0.019 | f4-12 0.269 | f11-12 n/a | f13-25 n/a; verdict ft1/v37_prestep_log.txt). Parent GO 21:04: stage A proceeds."
|
| 80 |
+
],
|
| 81 |
+
[
|
| 82 |
+
411.8,
|
| 83 |
+
"df: Filesystem Size Used Avail Use% Mounted on | /dev/loop2 20G 19G 724M 97% /kaggle/working | overlay 8.0T 6.9T 1.1T 87% / | "
|
| 84 |
+
],
|
| 85 |
+
[
|
| 86 |
+
411.8,
|
| 87 |
+
"kill-switch poller armed on lane ath-h2-finetune"
|
| 88 |
+
],
|
| 89 |
+
[
|
| 90 |
+
411.8,
|
| 91 |
+
"resume smoke: 2-iter warm-resume dry-run before stageA7 commit"
|
| 92 |
+
],
|
| 93 |
+
[
|
| 94 |
+
482.8,
|
| 95 |
+
"resume smoke: rc=0 iter markers=2 resume_line=True min_mean_length=9.45 (require >=3.0)"
|
| 96 |
+
],
|
| 97 |
+
[
|
| 98 |
+
483.8,
|
| 99 |
+
"resume smoke PASS - warm-resume path verified end-to-end, launching stageA7"
|
| 100 |
+
],
|
| 101 |
+
[
|
| 102 |
+
483.8,
|
| 103 |
+
"=== stageA7: ckpt /tmp/resume_a7/init.pt iters 1200 ==="
|
| 104 |
]
|
| 105 |
]
|
| 106 |
}
|