ath-h2-policies / ft1 /ft_progress.json
danielharkin21's picture
train smoke progress
11dc4a0 verified
Raw History Blame Contribute Delete
3.26 kB
{
"stage": "killed_by_switch",
"t0": 1790646666.9188678,
"events": [
[
1.2,
"kernel alive, py 3.12.13"
],
[
2.2,
"GPU: Tue Sep 29 01:51:08 2026 +-----------------------------------------------------------------------------------------+ | NVIDIA-SMI 580.159.04 Driver Version: 580.159.04 CUDA Version: 13.0 | +-----------------------------------------+------------------------+---"
],
[
13.3,
"GPU MODEL: Tesla T4, 580.159.04, 15360 MiB | Tesla T4, 580.159.04, 15360 MiB"
],
[
20.1,
"VULKAN: Processing triggers for man-db (2.10.2-1) ... | error: XDG_RUNTIME_DIR not set in the environment. | error: XDG_RUNTIME_DIR not set in the environment. | error: XDG_RUNTIME_DIR not set in the environment. | error: XDG_RUNTIME_DIR not set in the environment. | error: XDG_RUNTIME_DIR not set in the environment. | \tapiVersion = 4211000 (1.4.312) | \tdeviceName = Tesla T4 | \tapiVersion "
],
[
20.1,
"installing uv + python 3.11"
],
[
29.8,
"installing torch 2.7 + isaacsim 5.1 (large)"
],
[
253.2,
"post-isaacsim df: /dev/loop2 20G 16G 4.0G 80% /kaggle/working | "
],
[
253.2,
"cloning isaaclab v2.3.0"
],
[
273.0,
"cloning GR00T-WholeBodyControl (with lfs)"
],
[
331.4,
"downloading junsooki checkpoint + sample motions"
],
[
336.9,
"RESUME ckpt ft1/h2_medv_ft_last.pt (v36 final) - NO surgery (obs 1114 + std 0.15 already in ckpt)"
],
[
368.8,
"A6 warm checkpoint loaded; optimizer and adaptive sampler restored; trainer step reset"
],
[
368.8,
"v30: MEDV-ONLY motion set (no sample pkls) - 100% rally sampling"
],
[
369.5,
"medv_rally1 motionlib added (key-aligned name)"
],
[
370.0,
"real smpl file added (medv 50fps flat72)"
],
[
370.0,
"RSI patch: push_robot interval 0.02s, one impulse at motion_ts 3-5, bidirectional z and lateral; no altered reward or terminations"
],
[
370.0,
"check_environment.py --training"
],
[
376.2,
"IRAY LDD: ls: cannot access '/kaggle/working/v311/lib/python3.11/site-packages/isaacsim/extscache/omni.iray.lib': No such file or directory"
],
[
376.2,
"early-support-push preflight setup"
],
[
376.3,
"A6 reference and eval harness unchanged"
],
[
376.4,
"df: Filesystem Size Used Avail Use% Mounted on | /dev/loop2 20G 19G 724M 97% /kaggle/working | overlay 8.0T 6.9T 1.1T 87% / | "
],
[
376.4,
"kill-switch poller armed on lane ath-h2-finetune"
],
[
376.4,
"RSI preflight: 2 iters, 8 envs; no full tranche before event/survival evidence"
],
[
453.7,
"RSI preflight rc=0 mean_lengths=[9.2, 9.4] push_count=43 push_ts_range=3..5"
],
[
455.0,
"RSI preflight PASS; launch bounded A6-warm closed-loop perturbation trial"
],
[
455.0,
"=== rsiA6: ckpt /tmp/resume_a7/init.pt iters 1200 ==="
],
[
1395.2,
"live ckpt checkpointed to HF (455MB)"
],
[
2323.6,
"live ckpt checkpointed to HF (455MB)"
],
[
2971.8,
"kill poll: EXIT gen 23 rev 48925888d2c3d5cc808bb8d15f9af61321dac944 - LATCHED"
],
[
2971.8,
"KILL SWITCH EXIT received - stopping train, uploading state, exiting 42"
]
]
}