Upload batch 2: 13 files (7.18 GiB)
Browse files- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01400.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01500.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01600.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01700.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01800.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01900.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_02000.safetensors +3 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/swanlab/run-20260719_050608-8mjozf71/debug/debug.log +1 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/torchrun.log +279 -0
- checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/training_latest.pt +3 -0
- checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/launcher.nohup.log +0 -0
- checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/predictor_step_00100.safetensors +3 -0
- checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/predictor_step_00200.safetensors +3 -0
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01400.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:47dc357dbe9c48bf94c06cd37b97d9dc90a4d87caf423249551a1f447e74d61b
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01500.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:62511507db915d4d56937f88cc32450230abc4391a125236887715c8cad3eca1
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01600.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4434234f986e8aff06efe7607bb27b98e05bcd39bb157fe7941432fd3bbec5e
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01700.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:439a2c195d79c63524e02a71a2a0b9b17165a01c366828ff87f1662c96e1ec26
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01800.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8de414a009571876698aa394d148ff4f0b37af78f14d969d78a1cb5d5d07d4fc
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_01900.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bb120e07fd62f11ed04cc35299918acfd03f1b3949a619c97bb63fa6367dd7b9
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_02000.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:165402a63b55269c3163a257f2cb465f8fa3f677405f38c2e1e081b81159f911
|
| 3 |
+
size 654604408
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/swanlab/run-20260719_050608-8mjozf71/debug/debug.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Diagnostic log bound to file: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/swanlab/run-20260719_050608-8mjozf71/debug/debug.log
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/torchrun.log
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/hyworldplay_predictor_prefeature_v4/train_eval_manifest.jsonl",
|
| 3 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors",
|
| 4 |
+
"output_dir": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab",
|
| 5 |
+
"source_blocks": [
|
| 6 |
+
0,
|
| 7 |
+
53
|
| 8 |
+
],
|
| 9 |
+
"predictor_variant": "disca",
|
| 10 |
+
"max_steps": 2000,
|
| 11 |
+
"batch_size": 16,
|
| 12 |
+
"gradient_accumulation_steps": 1,
|
| 13 |
+
"block_warmup_steps": 100,
|
| 14 |
+
"scheduler_warmup_steps": 100,
|
| 15 |
+
"fusion_lr": 0.0001,
|
| 16 |
+
"blocks_lr": 1e-05,
|
| 17 |
+
"weight_decay": 0.01,
|
| 18 |
+
"hidden_weight": 0.1,
|
| 19 |
+
"velocity_weight": 1.0,
|
| 20 |
+
"grad_clip": 1.0,
|
| 21 |
+
"log_every": 10,
|
| 22 |
+
"save_every": 100,
|
| 23 |
+
"keep_snapshots": 20,
|
| 24 |
+
"seed": 0,
|
| 25 |
+
"num_workers": 0,
|
| 26 |
+
"max_records": null,
|
| 27 |
+
"bucket_by_context": false,
|
| 28 |
+
"gradient_checkpointing": true,
|
| 29 |
+
"save_optimizer_state": true,
|
| 30 |
+
"resume": null,
|
| 31 |
+
"swanlab_project": "HY-WorldPlay-Predictor-DisCa",
|
| 32 |
+
"swanlab_workspace": null,
|
| 33 |
+
"swanlab_experiment": "predictor-disca-0-53-b16x8-2000steps",
|
| 34 |
+
"swanlab_mode": "cloud",
|
| 35 |
+
"predictor_version": "predictor_disca",
|
| 36 |
+
"schema_version": "predictor_disca_text_kv_v1",
|
| 37 |
+
"supervision_pairs": [
|
| 38 |
+
[
|
| 39 |
+
0,
|
| 40 |
+
1
|
| 41 |
+
],
|
| 42 |
+
[
|
| 43 |
+
1,
|
| 44 |
+
2
|
| 45 |
+
],
|
| 46 |
+
[
|
| 47 |
+
2,
|
| 48 |
+
3
|
| 49 |
+
]
|
| 50 |
+
],
|
| 51 |
+
"records": 2800,
|
| 52 |
+
"dataset_pairs": 8400,
|
| 53 |
+
"steps_per_epoch_per_rank": 65,
|
| 54 |
+
"world_size": 8,
|
| 55 |
+
"per_rank_batch_size": 16,
|
| 56 |
+
"global_batch_size": 128,
|
| 57 |
+
"trainable_parameters": 176241152,
|
| 58 |
+
"frozen_projector_parameters": 0,
|
| 59 |
+
"dtype": "bfloat16",
|
| 60 |
+
"gpu": "NVIDIA H100 80GB HBM3"
|
| 61 |
+
}
|
| 62 |
+
swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use
|
| 63 |
+
`swanlab login --relogin` to force relogin
|
| 64 |
+
swanlab: Tracking run with swanlab version 0.9.0
|
| 65 |
+
swanlab: 💾 Run data saved at
|
| 66 |
+
/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/c
|
| 67 |
+
heckpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/swanlab/run-20260719_050608-8mjozf
|
| 68 |
+
71
|
| 69 |
+
swanlab: 📁 View project at
|
| 70 |
+
https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-DisCa
|
| 71 |
+
swanlab: 🚀 View run predictor-disca-0-53-b16x8-2000steps at
|
| 72 |
+
https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-DisCa/runs/8mjozf71
|
| 73 |
+
{"swanlab_run_id": "8mjozf71"}
|
| 74 |
+
{"avg_context_frames": 14.71875, "avg_micro_time_s": 6.3683194299992465, "epoch": 1, "global_step": 1, "grad_norm_max": 0.01446533203125, "hidden_mse": 0.016903931158594787, "loss": 0.014591015642508864, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 30.525243759155273, "velocity_mse": 0.012900622386951}
|
| 75 |
+
{"avg_context_frames": 14.180555555555555, "avg_micro_time_s": 1.6973763332352974, "epoch": 1, "global_step": 10, "grad_norm_max": 0.01385498046875, "hidden_mse": 0.016849822998564277, "loss": 0.013660540364475712, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011975558080141328}
|
| 76 |
+
{"avg_context_frames": 14.4, "avg_micro_time_s": 1.706131999749414, "epoch": 1, "global_step": 20, "grad_norm_max": 0.0108642578125, "hidden_mse": 0.017140438850037753, "loss": 0.014446394855622203, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.012732350960141048}
|
| 77 |
+
{"avg_context_frames": 14.26875, "avg_micro_time_s": 1.6902024415863708, "epoch": 1, "global_step": 30, "grad_norm_max": 0.01409912109375, "hidden_mse": 0.016241216578055172, "loss": 0.013358410890214147, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011734289216110484}
|
| 78 |
+
{"avg_context_frames": 14.134375, "avg_micro_time_s": 1.6864288714114082, "epoch": 1, "global_step": 40, "grad_norm_max": 0.01904296875, "hidden_mse": 0.01575567766558379, "loss": 0.013327148038661108, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011751580244163052}
|
| 79 |
+
{"avg_context_frames": 14.484375, "avg_micro_time_s": 1.6857911082748616, "epoch": 1, "global_step": 50, "grad_norm_max": 0.0234375, "hidden_mse": 0.01519326661946252, "loss": 0.012959708529524506, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011440381826832892}
|
| 80 |
+
{"avg_context_frames": 14.253125, "avg_micro_time_s": 1.6837490774865729, "epoch": 1, "global_step": 60, "grad_norm_max": 0.01708984375, "hidden_mse": 0.014341028686612845, "loss": 0.012393195397453384, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010959092515986413}
|
| 81 |
+
{"avg_context_frames": 14.234375, "avg_micro_time_s": 1.6833479166634788, "epoch": 2, "global_step": 70, "grad_norm_max": 0.0225830078125, "hidden_mse": 0.015120381687302143, "loss": 0.013338346488308162, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01182630832772702}
|
| 82 |
+
{"avg_context_frames": 14.403125000000001, "avg_micro_time_s": 1.6783960828634008, "epoch": 2, "global_step": 80, "grad_norm_max": 0.014404296875, "hidden_mse": 0.014159502333495767, "loss": 0.012736178463092074, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011320228286786006}
|
| 83 |
+
{"avg_context_frames": 14.053125000000001, "avg_micro_time_s": 1.6775284700990598, "epoch": 2, "global_step": 90, "grad_norm_max": 0.0281982421875, "hidden_mse": 0.01400059668812901, "loss": 0.012482481659390032, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011082421953324229}
|
| 84 |
+
{"avg_context_frames": 14.5625, "avg_micro_time_s": 1.678686482399644, "epoch": 2, "global_step": 100, "grad_norm_max": 0.01348876953125, "hidden_mse": 0.013929923344403506, "loss": 0.012565440696198494, "lr_blocks": 1e-05, "lr_fusion": 0.0001, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011172448418801651}
|
| 85 |
+
{"avg_context_frames": 14.175, "avg_micro_time_s": 1.7038727414888855, "epoch": 2, "global_step": 110, "grad_norm_max": 0.017822265625, "hidden_mse": 0.01416516478639096, "loss": 0.012822794512612746, "lr_blocks": 9.999316524962347e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011406278068898247}
|
| 86 |
+
{"avg_context_frames": 14.340625000000001, "avg_micro_time_s": 1.690337468202415, "epoch": 2, "global_step": 120, "grad_norm_max": 0.0233154296875, "hidden_mse": 0.013877624762244523, "loss": 0.012641063879709692, "lr_blocks": 9.99726628670463e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011253301409306006}
|
| 87 |
+
{"avg_context_frames": 14.09375, "avg_micro_time_s": 1.693360372800089, "epoch": 2, "global_step": 130, "grad_norm_max": 0.0167236328125, "hidden_mse": 0.014193043851992116, "loss": 0.013191334606381134, "lr_blocks": 9.993849845741525e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01177203023689799}
|
| 88 |
+
{"avg_context_frames": 14.20625, "avg_micro_time_s": 1.701355368750592, "epoch": 3, "global_step": 140, "grad_norm_max": 0.0213623046875, "hidden_mse": 0.013515560410451145, "loss": 0.012643122443114408, "lr_blocks": 9.989068136093873e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011291566424188205}
|
| 89 |
+
{"avg_context_frames": 14.30625, "avg_micro_time_s": 1.6922673213863164, "epoch": 3, "global_step": 150, "grad_norm_max": 0.0234375, "hidden_mse": 0.013559077697573231, "loss": 0.012912513717310503, "lr_blocks": 9.98292246503335e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011556605889927596}
|
| 90 |
+
{"avg_context_frames": 14.409375, "avg_micro_time_s": 1.6912575177138933, "epoch": 3, "global_step": 160, "grad_norm_max": 0.021728515625, "hidden_mse": 0.012874374637613074, "loss": 0.011889695067657158, "lr_blocks": 9.975414512725058e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010602257552091033}
|
| 91 |
+
{"avg_context_frames": 14.284375, "avg_micro_time_s": 1.6881116359512818, "epoch": 3, "global_step": 170, "grad_norm_max": 0.0235595703125, "hidden_mse": 0.013235451170476155, "loss": 0.012821374350460248, "lr_blocks": 9.966546331768192e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01149782930733636}
|
| 92 |
+
{"avg_context_frames": 14.55625, "avg_micro_time_s": 1.6881083145017328, "epoch": 3, "global_step": 180, "grad_norm_max": 0.0263671875, "hidden_mse": 0.01309632696211338, "loss": 0.01252620859304443, "lr_blocks": 9.956320346634877e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011216575902653859}
|
| 93 |
+
{"avg_context_frames": 14.293750000000001, "avg_micro_time_s": 1.6891175489261514, "epoch": 3, "global_step": 190, "grad_norm_max": 0.0162353515625, "hidden_mse": 0.012802221107995138, "loss": 0.012409800564637408, "lr_blocks": 9.944739353007344e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01112957840668969}
|
| 94 |
+
{"avg_context_frames": 13.793750000000001, "avg_micro_time_s": 1.6870502156383738, "epoch": 4, "global_step": 200, "grad_norm_max": 0.0150146484375, "hidden_mse": 0.012918339006137104, "loss": 0.012187451863428579, "lr_blocks": 9.931806517013612e-06, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010895617940695957}
|
| 95 |
+
{"avg_context_frames": 14.200000000000001, "avg_micro_time_s": 1.677001495748482, "epoch": 4, "global_step": 210, "grad_norm_max": 0.0166015625, "hidden_mse": 0.012752376409480348, "loss": 0.012700100988149644, "lr_blocks": 9.917525374361913e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.0114248632802628}
|
| 96 |
+
{"avg_context_frames": 14.387500000000001, "avg_micro_time_s": 1.6790696400625167, "epoch": 4, "global_step": 220, "grad_norm_max": 0.0252685546875, "hidden_mse": 0.012769673491129653, "loss": 0.012019666697597131, "lr_blocks": 9.901899829374048e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01074269930832088}
|
| 97 |
+
{"avg_context_frames": 14.134375, "avg_micro_time_s": 1.6766690597360139, "epoch": 4, "global_step": 230, "grad_norm_max": 0.021240234375, "hidden_mse": 0.01252085343003273, "loss": 0.012354082963429392, "lr_blocks": 9.884934153917998e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011101997626246885}
|
| 98 |
+
{"avg_context_frames": 14.5875, "avg_micro_time_s": 1.6781999800128689, "epoch": 4, "global_step": 240, "grad_norm_max": 0.0157470703125, "hidden_mse": 0.011941308557288722, "loss": 0.01208062154473737, "lr_blocks": 9.86663298624003e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010886490694247186}
|
| 99 |
+
{"avg_context_frames": 14.359375, "avg_micro_time_s": 1.672726408374365, "epoch": 4, "global_step": 250, "grad_norm_max": 0.01422119140625, "hidden_mse": 0.013191320811165497, "loss": 0.012905225792201236, "lr_blocks": 9.847001329696653e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011586093687219546}
|
| 100 |
+
{"avg_context_frames": 14.171875, "avg_micro_time_s": 1.6619861684026547, "epoch": 4, "global_step": 260, "grad_norm_max": 0.01611328125, "hidden_mse": 0.012081997399218381, "loss": 0.011730625136988238, "lr_blocks": 9.826044551386743e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010522425378439948}
|
| 101 |
+
{"avg_context_frames": 14.453125, "avg_micro_time_s": 1.6650776747635974, "epoch": 5, "global_step": 270, "grad_norm_max": 0.0198974609375, "hidden_mse": 0.012529171502683312, "loss": 0.012534417357528582, "lr_blocks": 9.803768380684242e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011281500186305493}
|
| 102 |
+
{"avg_context_frames": 14.53125, "avg_micro_time_s": 1.6554049376394688, "epoch": 5, "global_step": 280, "grad_norm_max": 0.016357421875, "hidden_mse": 0.012344440072774888, "loss": 0.012352285004453734, "lr_blocks": 9.780178907671788e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011117841006489472}
|
| 103 |
+
{"avg_context_frames": 14.084375000000001, "avg_micro_time_s": 1.6595323448000272, "epoch": 5, "global_step": 290, "grad_norm_max": 0.021240234375, "hidden_mse": 0.012424418586306274, "loss": 0.011968210444319994, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010725768603151664}
|
| 104 |
+
{"avg_context_frames": 14.171875, "avg_micro_time_s": 1.6620940328637517, "epoch": 5, "global_step": 300, "grad_norm_max": 0.0185546875, "hidden_mse": 0.01225031307549216, "loss": 0.012034611008130015, "lr_blocks": 9.729086208503174e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010809579689521343}
|
| 105 |
+
{"avg_context_frames": 14.259375, "avg_micro_time_s": 1.6782162721123315, "epoch": 5, "global_step": 310, "grad_norm_max": 0.02197265625, "hidden_mse": 0.01193973511690274, "loss": 0.011788620107108728, "lr_blocks": 9.701596950580807e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010594646586105228}
|
| 106 |
+
{"avg_context_frames": 14.334375000000001, "avg_micro_time_s": 1.6686666865873123, "epoch": 5, "global_step": 320, "grad_norm_max": 0.0155029296875, "hidden_mse": 0.012210708390921355, "loss": 0.01247087437659502, "lr_blocks": 9.672822322997305e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011249803507234903}
|
| 107 |
+
{"avg_context_frames": 14.221875, "avg_micro_time_s": 1.6616732870981652, "epoch": 6, "global_step": 330, "grad_norm_max": 0.014892578125, "hidden_mse": 0.012031648849369959, "loss": 0.011725777987157926, "lr_blocks": 9.642770192448537e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010522613144712523}
|
| 108 |
+
{"avg_context_frames": 14.475000000000001, "avg_micro_time_s": 1.663249169124174, "epoch": 6, "global_step": 340, "grad_norm_max": 0.0179443359375, "hidden_mse": 0.01219728536088951, "loss": 0.012152914865873755, "lr_blocks": 9.611448774886925e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01093318634084426}
|
| 109 |
+
{"avg_context_frames": 13.825000000000001, "avg_micro_time_s": 1.6703412917369862, "epoch": 6, "global_step": 350, "grad_norm_max": 0.01373291015625, "hidden_mse": 0.012240116082830356, "loss": 0.012266952311620116, "lr_blocks": 9.578866633275289e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01104294071556069}
|
| 110 |
+
{"avg_context_frames": 14.3, "avg_micro_time_s": 1.6716963182378095, "epoch": 6, "global_step": 360, "grad_norm_max": 0.01300048828125, "hidden_mse": 0.012173148326110095, "loss": 0.012153307441622019, "lr_blocks": 9.545032675245814e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010935992584563792}
|
| 111 |
+
{"avg_context_frames": 14.293750000000001, "avg_micro_time_s": 1.6808232290612068, "epoch": 6, "global_step": 370, "grad_norm_max": 0.01397705078125, "hidden_mse": 0.011603472143178806, "loss": 0.011518231267109514, "lr_blocks": 9.509956150664796e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010357884061522783}
|
| 112 |
+
{"avg_context_frames": 14.615625000000001, "avg_micro_time_s": 1.6778030030618538, "epoch": 6, "global_step": 380, "grad_norm_max": 0.02099609375, "hidden_mse": 0.01206234356504865, "loss": 0.01220254950458184, "lr_blocks": 9.473646649103819e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010996315145166592}
|
| 113 |
+
{"avg_context_frames": 14.175, "avg_micro_time_s": 1.6769782853614743, "epoch": 6, "global_step": 390, "grad_norm_max": 0.015869140625, "hidden_mse": 0.011909931572154165, "loss": 0.011906098818872125, "lr_blocks": 9.43611409721806e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010715105681447313}
|
| 114 |
+
{"avg_context_frames": 14.318750000000001, "avg_micro_time_s": 1.6752818771481544, "epoch": 7, "global_step": 400, "grad_norm_max": 0.01300048828125, "hidden_mse": 0.011655999638605863, "loss": 0.011750627792207525, "lr_blocks": 9.397368756032445e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010585027851630003}
|
| 115 |
+
{"avg_context_frames": 14.328125, "avg_micro_time_s": 1.677450421390313, "epoch": 7, "global_step": 410, "grad_norm_max": 0.0223388671875, "hidden_mse": 0.01190967960865237, "loss": 0.011954425496514888, "lr_blocks": 9.357421218136387e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010763457557186485}
|
| 116 |
+
{"avg_context_frames": 14.20625, "avg_micro_time_s": 1.6733501724360396, "epoch": 7, "global_step": 420, "grad_norm_max": 0.01556396484375, "hidden_mse": 0.011672888038447127, "loss": 0.011563985573593527, "lr_blocks": 9.31628240478787e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010396696761017666}
|
| 117 |
+
{"avg_context_frames": 14.321875, "avg_micro_time_s": 1.6728075357379568, "epoch": 7, "global_step": 430, "grad_norm_max": 0.0186767578125, "hidden_mse": 0.011731227574637161, "loss": 0.011881092737894505, "lr_blocks": 9.273963562927695e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01070796991698444}
|
| 118 |
+
{"avg_context_frames": 14.159375, "avg_micro_time_s": 1.6735653286392336, "epoch": 7, "global_step": 440, "grad_norm_max": 0.015869140625, "hidden_mse": 0.011862598883453757, "loss": 0.01177286847960204, "lr_blocks": 9.230476262104678e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010586608608718961}
|
| 119 |
+
{"avg_context_frames": 14.318750000000001, "avg_micro_time_s": 1.6664162646746263, "epoch": 7, "global_step": 450, "grad_norm_max": 0.0206298828125, "hidden_mse": 0.012222287594340742, "loss": 0.012133395049022512, "lr_blocks": 9.185832391312644e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010911166266305374}
|
| 120 |
+
{"avg_context_frames": 14.3, "avg_micro_time_s": 1.6672153513387458, "epoch": 8, "global_step": 460, "grad_norm_max": 0.0126953125, "hidden_mse": 0.011892647773493081, "loss": 0.012170521350344644, "lr_blocks": 9.140044155740102e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010981256602099166}
|
| 121 |
+
{"avg_context_frames": 14.3625, "avg_micro_time_s": 1.66273293293998, "epoch": 8, "global_step": 470, "grad_norm_max": 0.019775390625, "hidden_mse": 0.011298886651638897, "loss": 0.011162232892820612, "lr_blocks": 9.093124073433464e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010032344225328416}
|
| 122 |
+
{"avg_context_frames": 14.3125, "avg_micro_time_s": 1.6722241195380776, "epoch": 8, "global_step": 480, "grad_norm_max": 0.0159912109375, "hidden_mse": 0.011623429507017136, "loss": 0.011663176794536413, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010500833892729135}
|
| 123 |
+
{"avg_context_frames": 14.26875, "avg_micro_time_s": 1.6604970885855437, "epoch": 8, "global_step": 490, "grad_norm_max": 0.019775390625, "hidden_mse": 0.0121043776802253, "loss": 0.012028527632355691, "lr_blocks": 8.995939984474624e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010818089940585196}
|
| 124 |
+
{"avg_context_frames": 14.175, "avg_micro_time_s": 1.671721852789051, "epoch": 8, "global_step": 500, "grad_norm_max": 0.0252685546875, "hidden_mse": 0.01200883725541644, "loss": 0.012602345040068031, "lr_blocks": 8.94570254698197e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011401461373316124}
|
| 125 |
+
{"avg_context_frames": 14.515625, "avg_micro_time_s": 1.66979133807763, "epoch": 8, "global_step": 510, "grad_norm_max": 0.01287841796875, "hidden_mse": 0.011961949995020406, "loss": 0.011663943075109274, "lr_blocks": 8.894386393810563e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010467748151859269}
|
| 126 |
+
{"avg_context_frames": 14.125, "avg_micro_time_s": 1.6650106007375145, "epoch": 8, "global_step": 520, "grad_norm_max": 0.01446533203125, "hidden_mse": 0.012165928492322564, "loss": 0.012186324427602813, "lr_blocks": 8.842005554284296e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010969731508521364}
|
| 127 |
+
{"avg_context_frames": 14.421875, "avg_micro_time_s": 1.6684493572865904, "epoch": 9, "global_step": 530, "grad_norm_max": 0.016845703125, "hidden_mse": 0.011617637728340924, "loss": 0.011304739100160078, "lr_blocks": 8.788574348801676e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010142975387861953}
|
| 128 |
+
{"avg_context_frames": 14.340625000000001, "avg_micro_time_s": 1.6649726299736358, "epoch": 9, "global_step": 540, "grad_norm_max": 0.0213623046875, "hidden_mse": 0.012169572908896954, "loss": 0.012478833075147123, "lr_blocks": 8.734107384920771e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011261875747004525}
|
| 129 |
+
{"avg_context_frames": 14.025, "avg_micro_time_s": 1.6626613621527213, "epoch": 9, "global_step": 550, "grad_norm_max": 0.0189208984375, "hidden_mse": 0.011851134733296931, "loss": 0.011942122137406842, "lr_blocks": 8.67861955336566e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010757008683867753}
|
| 130 |
+
{"avg_context_frames": 14.496875000000001, "avg_micro_time_s": 1.6742402617110201, "epoch": 9, "global_step": 560, "grad_norm_max": 0.0191650390625, "hidden_mse": 0.011568118509603665, "loss": 0.01129704446066171, "lr_blocks": 8.622126023955446e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010140232573030518}
|
| 131 |
+
{"avg_context_frames": 14.231250000000001, "avg_micro_time_s": 1.6741461727222486, "epoch": 9, "global_step": 570, "grad_norm_max": 0.0146484375, "hidden_mse": 0.011797175847459585, "loss": 0.01185271711437963, "lr_blocks": 8.564642241456986e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010672999499365688}
|
| 132 |
+
{"avg_context_frames": 14.106250000000001, "avg_micro_time_s": 1.6786558078725649, "epoch": 9, "global_step": 580, "grad_norm_max": 0.0166015625, "hidden_mse": 0.011530527414288373, "loss": 0.011785041278926656, "lr_blocks": 8.506183921362443e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010631988535169513}
|
| 133 |
+
{"avg_context_frames": 14.543750000000001, "avg_micro_time_s": 1.6829396546374484, "epoch": 10, "global_step": 590, "grad_norm_max": 0.0235595703125, "hidden_mse": 0.011740072624525056, "loss": 0.011994902434526012, "lr_blocks": 8.446767045592829e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010820895194774494}
|
| 134 |
+
{"avg_context_frames": 14.109375, "avg_micro_time_s": 1.671282860262727, "epoch": 10, "global_step": 600, "grad_norm_max": 0.0191650390625, "hidden_mse": 0.011970765586011112, "loss": 0.012209405086468906, "lr_blocks": 8.386407858128707e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01101232846849598}
|
| 135 |
+
{"avg_context_frames": 14.225000000000001, "avg_micro_time_s": 1.6784421321997798, "epoch": 10, "global_step": 610, "grad_norm_max": 0.0146484375, "hidden_mse": 0.011483470286475495, "loss": 0.011486795824021102, "lr_blocks": 8.325122860569241e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010338448768015951}
|
| 136 |
+
{"avg_context_frames": 14.196875, "avg_micro_time_s": 1.675524012938331, "epoch": 10, "global_step": 620, "grad_norm_max": 0.01458740234375, "hidden_mse": 0.011668070487212391, "loss": 0.011516804079292342, "lr_blocks": 8.262928807620843e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010349997063167395}
|
| 137 |
+
{"avg_context_frames": 14.18125, "avg_micro_time_s": 1.6747926779120463, "epoch": 10, "global_step": 630, "grad_norm_max": 0.0135498046875, "hidden_mse": 0.011277910188073293, "loss": 0.011375020310515538, "lr_blocks": 8.199842702516584e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01024722931906581}
|
| 138 |
+
{"avg_context_frames": 14.459375000000001, "avg_micro_time_s": 1.6676438258135022, "epoch": 10, "global_step": 640, "grad_norm_max": 0.00994873046875, "hidden_mse": 0.011443477176362649, "loss": 0.011377838911721484, "lr_blocks": 8.135881792367686e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010233491146937014}
|
| 139 |
+
{"avg_context_frames": 14.428125000000001, "avg_micro_time_s": 1.670836907836565, "epoch": 10, "global_step": 650, "grad_norm_max": 0.01165771484375, "hidden_mse": 0.011770792270544917, "loss": 0.011601047770818697, "lr_blocks": 8.071063563448341e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010423968464601786}
|
| 140 |
+
{"avg_context_frames": 14.246875000000001, "avg_micro_time_s": 1.6668739861495852, "epoch": 11, "global_step": 660, "grad_norm_max": 0.01458740234375, "hidden_mse": 0.01172603099839762, "loss": 0.011920786584960298, "lr_blocks": 8.005405736415127e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010748183535179125}
|
| 141 |
+
{"avg_context_frames": 14.162500000000001, "avg_micro_time_s": 1.657212829864875, "epoch": 11, "global_step": 670, "grad_norm_max": 0.022216796875, "hidden_mse": 0.011572300613624976, "loss": 0.011661209503654391, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010503979376517238}
|
| 142 |
+
{"avg_context_frames": 14.271875000000001, "avg_micro_time_s": 1.6704402020586713, "epoch": 11, "global_step": 680, "grad_norm_max": 0.0166015625, "hidden_mse": 0.011474248359445483, "loss": 0.011235295224469156, "lr_blocks": 7.871643313414718e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010087870381539688}
|
| 143 |
+
{"avg_context_frames": 14.490625000000001, "avg_micro_time_s": 1.672706609874149, "epoch": 11, "global_step": 690, "grad_norm_max": 0.0152587890625, "hidden_mse": 0.01137218649382703, "loss": 0.011457442858954892, "lr_blocks": 7.803575286758365e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010320224193856121}
|
| 144 |
+
{"avg_context_frames": 14.171875, "avg_micro_time_s": 1.6699243164744986, "epoch": 11, "global_step": 700, "grad_norm_max": 0.01275634765625, "hidden_mse": 0.011979967611841858, "loss": 0.011598831223091112, "lr_blocks": 7.734740790612137e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010400834411848338}
|
| 145 |
+
{"avg_context_frames": 14.359375, "avg_micro_time_s": 1.6610987717267562, "epoch": 11, "global_step": 710, "grad_norm_max": 0.0164794921875, "hidden_mse": 0.01139996346901171, "loss": 0.011594607250299305, "lr_blocks": 7.66515864363997e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010454610927263276}
|
| 146 |
+
{"avg_context_frames": 14.24375, "avg_micro_time_s": 1.6625836039147543, "epoch": 12, "global_step": 720, "grad_norm_max": 0.022705078125, "hidden_mse": 0.011793063330696896, "loss": 0.012230632442515344, "lr_blocks": 7.594847868906076e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011051326082088054}
|
| 147 |
+
{"avg_context_frames": 14.387500000000001, "avg_micro_time_s": 1.6557012428886084, "epoch": 12, "global_step": 730, "grad_norm_max": 0.0216064453125, "hidden_mse": 0.011752536008134485, "loss": 0.011842861230252312, "lr_blocks": 7.52382768867422e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010667607601499186}
|
| 148 |
+
{"avg_context_frames": 14.40625, "avg_micro_time_s": 1.666427310848667, "epoch": 12, "global_step": 740, "grad_norm_max": 0.0137939453125, "hidden_mse": 0.011772934650070966, "loss": 0.011801917297998444, "lr_blocks": 7.452117519152542e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010624623831245118}
|
| 149 |
+
{"avg_context_frames": 14.381250000000001, "avg_micro_time_s": 1.6696053218744056, "epoch": 12, "global_step": 750, "grad_norm_max": 0.0130615234375, "hidden_mse": 0.01138463456882164, "loss": 0.011473681469215081, "lr_blocks": 7.379736965185369e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010335217957617715}
|
| 150 |
+
{"avg_context_frames": 14.01875, "avg_micro_time_s": 1.6625237152617047, "epoch": 12, "global_step": 760, "grad_norm_max": 0.0113525390625, "hidden_mse": 0.011510639847256244, "loss": 0.011572442314354704, "lr_blocks": 7.30670581489344e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010421378299361095}
|
| 151 |
+
{"avg_context_frames": 14.1875, "avg_micro_time_s": 1.6770685269762908, "epoch": 12, "global_step": 770, "grad_norm_max": 0.012451171875, "hidden_mse": 0.01137546225800179, "loss": 0.011072655598400162, "lr_blocks": 7.233044034264034e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.0099351093696896}
|
| 152 |
+
{"avg_context_frames": 14.3625, "avg_micro_time_s": 1.680464291626413, "epoch": 12, "global_step": 780, "grad_norm_max": 0.0172119140625, "hidden_mse": 0.011443223495734857, "loss": 0.011376247013686225, "lr_blocks": 7.158771761692464e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010231924639083446}
|
| 153 |
+
{"avg_context_frames": 14.11875, "avg_micro_time_s": 1.668213271964487, "epoch": 13, "global_step": 790, "grad_norm_max": 0.0157470703125, "hidden_mse": 0.011763761215843261, "loss": 0.012203776161186397, "lr_blocks": 7.083909302476453e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011027400114107877}
|
| 154 |
+
{"avg_context_frames": 14.096875, "avg_micro_time_s": 1.6663366743512598, "epoch": 13, "global_step": 800, "grad_norm_max": 0.0194091796875, "hidden_mse": 0.011545737728010864, "loss": 0.011326604709029199, "lr_blocks": 7.008477123264849e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01017203088849783}
|
| 155 |
+
{"avg_context_frames": 14.34375, "avg_micro_time_s": 1.6614910699252505, "epoch": 13, "global_step": 810, "grad_norm_max": 0.01507568359375, "hidden_mse": 0.01122304906602949, "loss": 0.011121913307579235, "lr_blocks": 6.932495846462262e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.0099996083881706}
|
| 156 |
+
{"avg_context_frames": 14.509375, "avg_micro_time_s": 1.6837449414124421, "epoch": 13, "global_step": 820, "grad_norm_max": 0.022216796875, "hidden_mse": 0.011082636169157924, "loss": 0.010782301204744727, "lr_blocks": 6.855986244591104e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009674037591321395}
|
| 157 |
+
{"avg_context_frames": 14.603125, "avg_micro_time_s": 1.6708717612764303, "epoch": 13, "global_step": 830, "grad_norm_max": 0.017578125, "hidden_mse": 0.011430756392655894, "loss": 0.01166371992439963, "lr_blocks": 6.778969234612583e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010520644317148255}
|
| 158 |
+
{"avg_context_frames": 14.296875, "avg_micro_time_s": 1.6763124754739693, "epoch": 13, "global_step": 840, "grad_norm_max": 0.012939453125, "hidden_mse": 0.011768269899766893, "loss": 0.011903852509567514, "lr_blocks": 6.701465872208216e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010727025504456833}
|
| 159 |
+
{"avg_context_frames": 13.93125, "avg_micro_time_s": 1.6694887150475552, "epoch": 14, "global_step": 850, "grad_norm_max": 0.017333984375, "hidden_mse": 0.0117756710620597, "loss": 0.011661562300287188, "lr_blocks": 6.6234973460234184e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010483995190588758}
|
| 160 |
+
{"avg_context_frames": 14.36875, "avg_micro_time_s": 1.6700406820895297, "epoch": 14, "global_step": 860, "grad_norm_max": 0.016357421875, "hidden_mse": 0.010869661986362189, "loss": 0.010810558591037989, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009723592398222536}
|
| 161 |
+
{"avg_context_frames": 14.240625000000001, "avg_micro_time_s": 1.654148106536377, "epoch": 14, "global_step": 870, "grad_norm_max": 0.020751953125, "hidden_mse": 0.011577105836477131, "loss": 0.011544856429100038, "lr_blocks": 6.466250186922325e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010387145873392001}
|
| 162 |
+
{"avg_context_frames": 14.65625, "avg_micro_time_s": 1.6614088644604637, "epoch": 14, "global_step": 880, "grad_norm_max": 0.01531982421875, "hidden_mse": 0.011320531176170335, "loss": 0.01140951054985635, "lr_blocks": 6.387014543809224e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010277457459596918}
|
| 163 |
+
{"avg_context_frames": 14.146875000000001, "avg_micro_time_s": 1.6693167886885931, "epoch": 14, "global_step": 890, "grad_norm_max": 0.0108642578125, "hidden_mse": 0.01169083885033615, "loss": 0.011576085269916804, "lr_blocks": 6.3073997047691e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010407001402927564}
|
| 164 |
+
{"avg_context_frames": 14.21875, "avg_micro_time_s": 1.6690942833494775, "epoch": 14, "global_step": 900, "grad_norm_max": 0.016357421875, "hidden_mse": 0.011695230467012152, "loss": 0.011713925638468937, "lr_blocks": 6.227427435703997e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01054440257139504}
|
| 165 |
+
{"avg_context_frames": 14.293750000000001, "avg_micro_time_s": 1.6668594162874797, "epoch": 14, "global_step": 910, "grad_norm_max": 0.0142822265625, "hidden_mse": 0.01127870319178328, "loss": 0.011360114079434425, "lr_blocks": 6.147119600233758e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010232243849895895}
|
| 166 |
+
{"avg_context_frames": 14.275, "avg_micro_time_s": 1.6744390483105236, "epoch": 15, "global_step": 920, "grad_norm_max": 0.015380859375, "hidden_mse": 0.01180309365154244, "loss": 0.011988794663920999, "lr_blocks": 6.066498153718735e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010808485298184678}
|
| 167 |
+
{"avg_context_frames": 14.025, "avg_micro_time_s": 1.660698739412692, "epoch": 15, "global_step": 930, "grad_norm_max": 0.011962890625, "hidden_mse": 0.011142635776195675, "loss": 0.011125632375478745, "lr_blocks": 5.985585137257401e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010011368786217646}
|
| 168 |
+
{"avg_context_frames": 14.1875, "avg_micro_time_s": 1.6714014525521634, "epoch": 15, "global_step": 940, "grad_norm_max": 0.01141357421875, "hidden_mse": 0.011377936857752503, "loss": 0.011362285091308877, "lr_blocks": 5.904402671660551e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010224491398548708}
|
| 169 |
+
{"avg_context_frames": 14.471875, "avg_micro_time_s": 1.6828720892244746, "epoch": 15, "global_step": 950, "grad_norm_max": 0.015625, "hidden_mse": 0.011794780974742026, "loss": 0.011746052757371218, "lr_blocks": 5.82297295140367e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010566574707627297}
|
| 170 |
+
{"avg_context_frames": 14.1875, "avg_micro_time_s": 1.6743098544498936, "epoch": 15, "global_step": 960, "grad_norm_max": 0.0147705078125, "hidden_mse": 0.011310246528591961, "loss": 0.01120361810317263, "lr_blocks": 5.74131823855921e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010072593454970047}
|
| 171 |
+
{"avg_context_frames": 14.421875, "avg_micro_time_s": 1.669582998111946, "epoch": 15, "global_step": 970, "grad_norm_max": 0.0167236328125, "hidden_mse": 0.011059153161477299, "loss": 0.010959938418818638, "lr_blocks": 5.659460856710346e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009854023071238772}
|
| 172 |
+
{"avg_context_frames": 14.106250000000001, "avg_micro_time_s": 1.667396959274629, "epoch": 16, "global_step": 980, "grad_norm_max": 0.01483154296875, "hidden_mse": 0.011709797941148282, "loss": 0.012077316356590019, "lr_blocks": 5.577423184847932e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010906336503103376}
|
| 173 |
+
{"avg_context_frames": 14.381250000000001, "avg_micro_time_s": 1.6778730853613526, "epoch": 16, "global_step": 990, "grad_norm_max": 0.0115966796875, "hidden_mse": 0.011467087210621686, "loss": 0.011327883088961245, "lr_blocks": 5.495227651252315e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01018117435160093}
|
| 174 |
+
{"avg_context_frames": 14.543750000000001, "avg_micro_time_s": 1.6817755866748485, "epoch": 16, "global_step": 1000, "grad_norm_max": 0.01263427734375, "hidden_mse": 0.010680826468160376, "loss": 0.010792405833490193, "lr_blocks": 5.412896727361663e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.00972432320122607}
|
| 175 |
+
{"avg_context_frames": 14.240625000000001, "avg_micro_time_s": 1.661411627598136, "epoch": 16, "global_step": 1010, "grad_norm_max": 0.0185546875, "hidden_mse": 0.011387520033167675, "loss": 0.011222705629188569, "lr_blocks": 5.3304529216284974e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010083953657886014}
|
| 176 |
+
{"avg_context_frames": 14.234375, "avg_micro_time_s": 1.6655827227739792, "epoch": 16, "global_step": 1020, "grad_norm_max": 0.014892578125, "hidden_mse": 0.011972595425322653, "loss": 0.011918087041703985, "lr_blocks": 5.247918773366112e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010720827511977405}
|
| 177 |
+
{"avg_context_frames": 14.240625000000001, "avg_micro_time_s": 1.6604705360390655, "epoch": 16, "global_step": 1030, "grad_norm_max": 0.01165771484375, "hidden_mse": 0.011471667175646872, "loss": 0.011304579180432484, "lr_blocks": 5.165316846586541e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010157412447733806}
|
| 178 |
+
{"avg_context_frames": 14.378125, "avg_micro_time_s": 1.6618204411388433, "epoch": 16, "global_step": 1040, "grad_norm_max": 0.01373291015625, "hidden_mse": 0.011016206297790633, "loss": 0.011066284886328505, "lr_blocks": 5.082669723831793e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009964664198923856}
|
| 179 |
+
{"avg_context_frames": 14.165625, "avg_micro_time_s": 1.6536375549985678, "epoch": 17, "global_step": 1050, "grad_norm_max": 0.01190185546875, "hidden_mse": 0.011110378883313388, "loss": 0.010883377410937102, "lr_blocks": 5e-06, "lr_fusion": 5e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009772339568007738}
|
| 180 |
+
{"avg_context_frames": 14.240625000000001, "avg_micro_time_s": 1.6610624320619536, "epoch": 17, "global_step": 1060, "grad_norm_max": 0.01287841796875, "hidden_mse": 0.011551116046030076, "loss": 0.011778575333300978, "lr_blocks": 4.917330276168208e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010623463714728132}
|
| 181 |
+
{"avg_context_frames": 14.293750000000001, "avg_micro_time_s": 1.6562152313763363, "epoch": 17, "global_step": 1070, "grad_norm_max": 0.01031494140625, "hidden_mse": 0.01174976057955064, "loss": 0.011603503639344127, "lr_blocks": 4.8346831534134595e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010428527591284365}
|
| 182 |
+
{"avg_context_frames": 14.30625, "avg_micro_time_s": 1.6629208340753394, "epoch": 17, "global_step": 1080, "grad_norm_max": 0.018798828125, "hidden_mse": 0.011370424262713641, "loss": 0.011329973622923718, "lr_blocks": 4.752081226633888e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010192931210622192}
|
| 183 |
+
{"avg_context_frames": 14.396875000000001, "avg_micro_time_s": 1.6774043477347733, "epoch": 17, "global_step": 1090, "grad_norm_max": 0.01385498046875, "hidden_mse": 0.011211473296862097, "loss": 0.011175840767100454, "lr_blocks": 4.669547078371503e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010054693405982107}
|
| 184 |
+
{"avg_context_frames": 14.190625, "avg_micro_time_s": 1.6634978182628402, "epoch": 17, "global_step": 1100, "grad_norm_max": 0.0118408203125, "hidden_mse": 0.011585666466271505, "loss": 0.01196135599166155, "lr_blocks": 4.587103272638339e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010802789340959863}
|
| 185 |
+
{"avg_context_frames": 14.4375, "avg_micro_time_s": 1.6532553265013121, "epoch": 18, "global_step": 1110, "grad_norm_max": 0.01300048828125, "hidden_mse": 0.010795156040694566, "loss": 0.010537886351812631, "lr_blocks": 4.504772348747687e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009458370739594102}
|
| 186 |
+
{"avg_context_frames": 13.996875000000001, "avg_micro_time_s": 1.6608335784279917, "epoch": 18, "global_step": 1120, "grad_norm_max": 0.01458740234375, "hidden_mse": 0.01167248976416886, "loss": 0.011236506706336514, "lr_blocks": 4.42257681515207e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010069257783470675}
|
| 187 |
+
{"avg_context_frames": 14.21875, "avg_micro_time_s": 1.6740720257494104, "epoch": 18, "global_step": 1130, "grad_norm_max": 0.00994873046875, "hidden_mse": 0.01124509716173634, "loss": 0.011494273459538818, "lr_blocks": 4.340539143289655e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010369763767812401}
|
| 188 |
+
{"avg_context_frames": 14.287500000000001, "avg_micro_time_s": 1.6676692341370654, "epoch": 18, "global_step": 1140, "grad_norm_max": 0.013427734375, "hidden_mse": 0.011781077337218449, "loss": 0.011909169994760306, "lr_blocks": 4.25868176144079e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010731062246486546}
|
| 189 |
+
{"avg_context_frames": 14.178125000000001, "avg_micro_time_s": 1.6677437865000684, "epoch": 18, "global_step": 1150, "grad_norm_max": 0.01025390625, "hidden_mse": 0.011095615645172075, "loss": 0.011239823908545077, "lr_blocks": 4.17702704859633e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010130262339953334}
|
| 190 |
+
{"avg_context_frames": 14.415625, "avg_micro_time_s": 1.6578270324760525, "epoch": 18, "global_step": 1160, "grad_norm_max": 0.0189208984375, "hidden_mse": 0.011448635597480461, "loss": 0.0112959076941479, "lr_blocks": 4.0955973283394525e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010151044093072416}
|
| 191 |
+
{"avg_context_frames": 14.553125000000001, "avg_micro_time_s": 1.6594081541130437, "epoch": 18, "global_step": 1170, "grad_norm_max": 0.0125732421875, "hidden_mse": 0.011061798536684365, "loss": 0.01104984058183618, "lr_blocks": 4.0144148627426e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.00994366078521125}
|
| 192 |
+
{"avg_context_frames": 14.421875, "avg_micro_time_s": 1.6631646836503933, "epoch": 19, "global_step": 1180, "grad_norm_max": 0.01141357421875, "hidden_mse": 0.01066333387279883, "loss": 0.010685365134850145, "lr_blocks": 3.9335018462812664e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009619031671900303}
|
| 193 |
+
{"avg_context_frames": 14.253125, "avg_micro_time_s": 1.6555977173513385, "epoch": 19, "global_step": 1190, "grad_norm_max": 0.0111083984375, "hidden_mse": 0.011523925105575473, "loss": 0.011254557583015412, "lr_blocks": 3.852880399766243e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010102165024727584}
|
| 194 |
+
{"avg_context_frames": 13.9625, "avg_micro_time_s": 1.6477898450761133, "epoch": 19, "global_step": 1200, "grad_norm_max": 0.0140380859375, "hidden_mse": 0.011714816611493007, "loss": 0.011656381204375066, "lr_blocks": 3.7725725642960047e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010484899516450243}
|
| 195 |
+
{"avg_context_frames": 14.340625000000001, "avg_micro_time_s": 1.6606175010983861, "epoch": 19, "global_step": 1210, "grad_norm_max": 0.013916015625, "hidden_mse": 0.011326234409352764, "loss": 0.01120215451810509, "lr_blocks": 3.6926002952309015e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010069531074259432}
|
| 196 |
+
{"avg_context_frames": 14.49375, "avg_micro_time_s": 1.6539430881886803, "epoch": 19, "global_step": 1220, "grad_norm_max": 0.01275634765625, "hidden_mse": 0.011486595304450021, "loss": 0.01146072629489936, "lr_blocks": 3.6129854561907786e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01031206674524583}
|
| 197 |
+
{"avg_context_frames": 14.39375, "avg_micro_time_s": 1.6584447426888802, "epoch": 19, "global_step": 1230, "grad_norm_max": 0.012451171875, "hidden_mse": 0.010983252251753584, "loss": 0.010907892114482821, "lr_blocks": 3.533749813077677e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009809566976036877}
|
| 198 |
+
{"avg_context_frames": 14.100000000000001, "avg_micro_time_s": 1.656307424226907, "epoch": 20, "global_step": 1240, "grad_norm_max": 0.01263427734375, "hidden_mse": 0.011136419005924837, "loss": 0.011541601398494096, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010427959496155381}
|
| 199 |
+
{"avg_context_frames": 14.168750000000001, "avg_micro_time_s": 1.6521637100984663, "epoch": 20, "global_step": 1250, "grad_norm_max": 0.012451171875, "hidden_mse": 0.011190626042662188, "loss": 0.01142194886924699, "lr_blocks": 3.3765026539765832e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010302886262070389}
|
| 200 |
+
{"avg_context_frames": 14.609375, "avg_micro_time_s": 1.6508932352240664, "epoch": 20, "global_step": 1260, "grad_norm_max": 0.01141357421875, "hidden_mse": 0.011175389477284626, "loss": 0.011005451972596348, "lr_blocks": 3.298534127791785e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009887913009151816}
|
| 201 |
+
{"avg_context_frames": 14.15625, "avg_micro_time_s": 1.6523738718875394, "epoch": 20, "global_step": 1270, "grad_norm_max": 0.01092529296875, "hidden_mse": 0.011317605827935041, "loss": 0.011162525363033637, "lr_blocks": 3.2210307653874175e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010030764807015658}
|
| 202 |
+
{"avg_context_frames": 14.256250000000001, "avg_micro_time_s": 1.6502642466119142, "epoch": 20, "global_step": 1280, "grad_norm_max": 0.01080322265625, "hidden_mse": 0.011835476104170085, "loss": 0.011668690148508177, "lr_blocks": 3.1440137554088957e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010485142481047661}
|
| 203 |
+
{"avg_context_frames": 14.240625000000001, "avg_micro_time_s": 1.6596468387386267, "epoch": 20, "global_step": 1290, "grad_norm_max": 0.009521484375, "hidden_mse": 0.011371044552652166, "loss": 0.011234781093662605, "lr_blocks": 3.06750415353774e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010097676678560675}
|
| 204 |
+
{"avg_context_frames": 14.390625, "avg_micro_time_s": 1.652364479660173, "epoch": 20, "global_step": 1300, "grad_norm_max": 0.0113525390625, "hidden_mse": 0.01134974528104067, "loss": 0.011419776943512261, "lr_blocks": 2.991522876735154e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010284802451496945}
|
| 205 |
+
{"avg_context_frames": 14.2375, "avg_micro_time_s": 1.6571869058017912, "epoch": 21, "global_step": 1310, "grad_norm_max": 0.015625, "hidden_mse": 0.011261974036460743, "loss": 0.011258097348036246, "lr_blocks": 2.9160906975235493e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010131899925181643}
|
| 206 |
+
{"avg_context_frames": 14.3625, "avg_micro_time_s": 1.646345046314673, "epoch": 21, "global_step": 1320, "grad_norm_max": 0.01043701171875, "hidden_mse": 0.010938315524253995, "loss": 0.010823095840169118, "lr_blocks": 2.8412282383075362e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009729264292400331}
|
| 207 |
+
{"avg_context_frames": 14.615625000000001, "avg_micro_time_s": 1.6541958835270636, "epoch": 21, "global_step": 1330, "grad_norm_max": 0.01177978515625, "hidden_mse": 0.011133276578038932, "loss": 0.011046531575266272, "lr_blocks": 2.766955965735968e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009933203872060405}
|
| 208 |
+
{"avg_context_frames": 13.946875, "avg_micro_time_s": 1.659248725712314, "epoch": 21, "global_step": 1340, "grad_norm_max": 0.00933837890625, "hidden_mse": 0.011702069418970496, "loss": 0.011699286807561294, "lr_blocks": 2.693294185106562e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010529079881962389}
|
| 209 |
+
{"avg_context_frames": 14.146875000000001, "avg_micro_time_s": 1.6543794904744573, "epoch": 21, "global_step": 1350, "grad_norm_max": 0.01434326171875, "hidden_mse": 0.010978883533971385, "loss": 0.01105476276134141, "lr_blocks": 2.6202630348146323e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009956874430645258}
|
| 210 |
+
{"avg_context_frames": 14.537500000000001, "avg_micro_time_s": 1.6427724638611836, "epoch": 21, "global_step": 1360, "grad_norm_max": 0.01416015625, "hidden_mse": 0.011333403328899296, "loss": 0.01119010701077059, "lr_blocks": 2.5478824808474613e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010056766687193887}
|
| 211 |
+
{"avg_context_frames": 14.3, "avg_micro_time_s": 1.649842259700381, "epoch": 22, "global_step": 1370, "grad_norm_max": 0.0091552734375, "hidden_mse": 0.011966329917777331, "loss": 0.01182466876343824, "lr_blocks": 2.476172311325783e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010628035763511436}
|
| 212 |
+
{"avg_context_frames": 14.484375, "avg_micro_time_s": 1.6768584499135615, "epoch": 22, "global_step": 1380, "grad_norm_max": 0.0103759765625, "hidden_mse": 0.01126847570994869, "loss": 0.011210237367777154, "lr_blocks": 2.4051521310939258e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010083389858482407}
|
| 213 |
+
{"avg_context_frames": 14.14375, "avg_micro_time_s": 1.6518969393000589, "epoch": 22, "global_step": 1390, "grad_norm_max": 0.01116943359375, "hidden_mse": 0.011465440806932748, "loss": 0.011482517694821582, "lr_blocks": 2.3348413563600324e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010335973603650928}
|
| 214 |
+
{"avg_context_frames": 13.96875, "avg_micro_time_s": 1.6484803539868154, "epoch": 22, "global_step": 1400, "grad_norm_max": 0.010498046875, "hidden_mse": 0.011556138604646549, "loss": 0.01106837273691781, "lr_blocks": 2.265259209387867e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009912758847349323}
|
| 215 |
+
{"avg_context_frames": 14.165625, "avg_micro_time_s": 1.6538986773754005, "epoch": 22, "global_step": 1410, "grad_norm_max": 0.009765625, "hidden_mse": 0.010561504488578066, "loss": 0.010613942006602884, "lr_blocks": 2.1964247132416373e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009557791572296992}
|
| 216 |
+
{"avg_context_frames": 14.371875000000001, "avg_micro_time_s": 1.6527693050378731, "epoch": 22, "global_step": 1420, "grad_norm_max": 0.0111083984375, "hidden_mse": 0.011084572208346799, "loss": 0.010916793579235674, "lr_blocks": 2.1283566865852824e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009808336396235974}
|
| 217 |
+
{"avg_context_frames": 14.471875, "avg_micro_time_s": 1.6431071163879096, "epoch": 22, "global_step": 1430, "grad_norm_max": 0.00958251953125, "hidden_mse": 0.011512590287020431, "loss": 0.011875563196372242, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010724304232280702}
|
| 218 |
+
{"avg_context_frames": 14.2125, "avg_micro_time_s": 1.6510546650504694, "epoch": 23, "global_step": 1440, "grad_norm_max": 0.0076904296875, "hidden_mse": 0.010945631068898365, "loss": 0.010957002855138854, "lr_blocks": 1.9945942635848745e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009862439712742345}
|
| 219 |
+
{"avg_context_frames": 13.9625, "avg_micro_time_s": 1.6461858294234846, "epoch": 23, "global_step": 1450, "grad_norm_max": 0.0069580078125, "hidden_mse": 0.01127517645363696, "loss": 0.011059572256635876, "lr_blocks": 1.928936436551661e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.0099320545559749}
|
| 220 |
+
{"avg_context_frames": 14.3625, "avg_micro_time_s": 1.6362899592495523, "epoch": 23, "global_step": 1460, "grad_norm_max": 0.01116943359375, "hidden_mse": 0.011928864050423727, "loss": 0.012370476470096037, "lr_blocks": 1.864118207632315e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.011177590105216951}
|
| 221 |
+
{"avg_context_frames": 14.5875, "avg_micro_time_s": 1.6395295883732617, "epoch": 23, "global_step": 1470, "grad_norm_max": 0.0087890625, "hidden_mse": 0.010839305043919013, "loss": 0.01071897276560776, "lr_blocks": 1.8001572974834169e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009635042271111161}
|
| 222 |
+
{"avg_context_frames": 14.340625000000001, "avg_micro_time_s": 1.6361626389869344, "epoch": 23, "global_step": 1480, "grad_norm_max": 0.01483154296875, "hidden_mse": 0.011283126019407064, "loss": 0.01097512152628042, "lr_blocks": 1.7370711923791567e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009846808918518947}
|
| 223 |
+
{"avg_context_frames": 14.428125000000001, "avg_micro_time_s": 1.639323206937843, "epoch": 23, "global_step": 1490, "grad_norm_max": 0.01043701171875, "hidden_mse": 0.01127969539957121, "loss": 0.011254827643278988, "lr_blocks": 1.6748771394307584e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010126858117291705}
|
| 224 |
+
{"avg_context_frames": 14.200000000000001, "avg_micro_time_s": 1.638041897965377, "epoch": 24, "global_step": 1500, "grad_norm_max": 0.009521484375, "hidden_mse": 0.011386207316536457, "loss": 0.011183572874870151, "lr_blocks": 1.6135921418712959e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010044952091993764}
|
| 225 |
+
{"avg_context_frames": 14.3625, "avg_micro_time_s": 1.6395215454758727, "epoch": 24, "global_step": 1510, "grad_norm_max": 0.00994873046875, "hidden_mse": 0.011413303896551952, "loss": 0.011490373039850966, "lr_blocks": 1.5532329544071712e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010349042672896758}
|
| 226 |
+
{"avg_context_frames": 14.496875000000001, "avg_micro_time_s": 1.6431810966249032, "epoch": 24, "global_step": 1520, "grad_norm_max": 0.006927490234375, "hidden_mse": 0.011631385050714017, "loss": 0.011430102051235735, "lr_blocks": 1.4938160786375571e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010266963561298326}
|
| 227 |
+
{"avg_context_frames": 13.903125000000001, "avg_micro_time_s": 1.665714266872965, "epoch": 24, "global_step": 1530, "grad_norm_max": 0.010009765625, "hidden_mse": 0.011071189382346348, "loss": 0.010993697366211565, "lr_blocks": 1.4353577585430152e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009886578464647756}
|
| 228 |
+
{"avg_context_frames": 14.146875000000001, "avg_micro_time_s": 1.6518768868489133, "epoch": 24, "global_step": 1540, "grad_norm_max": 0.00799560546875, "hidden_mse": 0.011262133251875639, "loss": 0.011162439128383994, "lr_blocks": 1.3778739760445552e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01003622582065873}
|
| 229 |
+
{"avg_context_frames": 14.296875, "avg_micro_time_s": 1.6510583013143334, "epoch": 24, "global_step": 1550, "grad_norm_max": 0.0084228515625, "hidden_mse": 0.01136539108119905, "loss": 0.011442213400732726, "lr_blocks": 1.321380446634342e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010305674298433588}
|
| 230 |
+
{"avg_context_frames": 14.434375000000001, "avg_micro_time_s": 1.6434962125258608, "epoch": 24, "global_step": 1560, "grad_norm_max": 0.00836181640625, "hidden_mse": 0.010991631337674335, "loss": 0.010861971409758553, "lr_blocks": 1.2658926150792321e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009762808290543035}
|
| 231 |
+
{"avg_context_frames": 14.578125, "avg_micro_time_s": 1.6481255472146588, "epoch": 25, "global_step": 1570, "grad_norm_max": 0.007354736328125, "hidden_mse": 0.011273362330393867, "loss": 0.01115232922602445, "lr_blocks": 1.2114256511983274e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010024992999387906}
|
| 232 |
+
{"avg_context_frames": 14.296875, "avg_micro_time_s": 1.6549864632390383, "epoch": 25, "global_step": 1580, "grad_norm_max": 0.00628662109375, "hidden_mse": 0.010917296970728785, "loss": 0.010790761729003862, "lr_blocks": 1.157994445715706e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009699032048229129}
|
| 233 |
+
{"avg_context_frames": 14.121875000000001, "avg_micro_time_s": 1.6433439763863136, "epoch": 25, "global_step": 1590, "grad_norm_max": 0.0072021484375, "hidden_mse": 0.011266779497964308, "loss": 0.01100723147392273, "lr_blocks": 1.1056136061894386e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009880553546827287}
|
| 234 |
+
{"avg_context_frames": 14.346875, "avg_micro_time_s": 1.6426967713901832, "epoch": 25, "global_step": 1600, "grad_norm_max": 0.00665283203125, "hidden_mse": 0.01106002515880391, "loss": 0.010793162672780455, "lr_blocks": 1.0542974530180327e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009687160170869902}
|
| 235 |
+
{"avg_context_frames": 14.18125, "avg_micro_time_s": 1.6422525028112434, "epoch": 25, "global_step": 1610, "grad_norm_max": 0.00830078125, "hidden_mse": 0.011610816064057873, "loss": 0.011938656150596217, "lr_blocks": 1.0040600155253766e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010777574474923314}
|
| 236 |
+
{"avg_context_frames": 14.24375, "avg_micro_time_s": 1.6406341752641311, "epoch": 25, "global_step": 1620, "grad_norm_max": 0.0068359375, "hidden_mse": 0.011537419148953633, "loss": 0.011604450049344451, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01045070806867443}
|
| 237 |
+
{"avg_context_frames": 14.15, "avg_micro_time_s": 1.6369994136250172, "epoch": 26, "global_step": 1630, "grad_norm_max": 0.006378173828125, "hidden_mse": 0.011142240453045816, "loss": 0.01115040056174621, "lr_blocks": 9.068759265665384e-07, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010036176547873766}
|
| 238 |
+
{"avg_context_frames": 14.28125, "avg_micro_time_s": 1.6424545892885363, "epoch": 26, "global_step": 1640, "grad_norm_max": 0.005523681640625, "hidden_mse": 0.011423144373111427, "loss": 0.011198497947771103, "lr_blocks": 8.599558442598998e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010056183492997662}
|
| 239 |
+
{"avg_context_frames": 14.381250000000001, "avg_micro_time_s": 1.6438027446994967, "epoch": 26, "global_step": 1650, "grad_norm_max": 0.0067138671875, "hidden_mse": 0.011288748204242438, "loss": 0.011151470913318917, "lr_blocks": 8.141676086873574e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010022596060298384}
|
| 240 |
+
{"avg_context_frames": 14.359375, "avg_micro_time_s": 1.6455117818499276, "epoch": 26, "global_step": 1660, "grad_norm_max": 0.00653076171875, "hidden_mse": 0.011075893428642304, "loss": 0.01110358367441222, "lr_blocks": 7.695237378953224e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009995994257042186}
|
| 241 |
+
{"avg_context_frames": 14.046875, "avg_micro_time_s": 1.6422561920637238, "epoch": 26, "global_step": 1670, "grad_norm_max": 0.0059814453125, "hidden_mse": 0.011189652676694096, "loss": 0.011105867236619816, "lr_blocks": 7.260364370723044e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009986901946831495}
|
| 242 |
+
{"avg_context_frames": 14.24375, "avg_micro_time_s": 1.6495943162015465, "epoch": 26, "global_step": 1680, "grad_norm_max": 0.006927490234375, "hidden_mse": 0.011674578482052313, "loss": 0.011711610347265379, "lr_blocks": 6.837175952121305e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010544152522925287}
|
| 243 |
+
{"avg_context_frames": 14.418750000000001, "avg_micro_time_s": 1.6613924485140161, "epoch": 26, "global_step": 1690, "grad_norm_max": 0.0054931640625, "hidden_mse": 0.011259417346445845, "loss": 0.011245386057998985, "lr_blocks": 6.425787818636131e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010119444341398776}
|
| 244 |
+
{"avg_context_frames": 14.05, "avg_micro_time_s": 1.6575991860758224, "epoch": 27, "global_step": 1700, "grad_norm_max": 0.00628662109375, "hidden_mse": 0.011074321490013972, "loss": 0.010853745066560805, "lr_blocks": 6.026312439675553e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009746312914649025}
|
| 245 |
+
{"avg_context_frames": 14.256250000000001, "avg_micro_time_s": 1.6475757559775959, "epoch": 27, "global_step": 1710, "grad_norm_max": 0.005950927734375, "hidden_mse": 0.01098624971928075, "loss": 0.01108252534759231, "lr_blocks": 5.63885902781941e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.00998390032327734}
|
| 246 |
+
{"avg_context_frames": 14.415625, "avg_micro_time_s": 1.643883330663448, "epoch": 27, "global_step": 1720, "grad_norm_max": 0.006591796875, "hidden_mse": 0.011700686026597396, "loss": 0.011627906281501055, "lr_blocks": 5.263533508961827e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010457837616559119}
|
| 247 |
+
{"avg_context_frames": 14.109375, "avg_micro_time_s": 1.6416245827273088, "epoch": 27, "global_step": 1730, "grad_norm_max": 0.0067138671875, "hidden_mse": 0.011639733344782145, "loss": 0.011623957718256861, "lr_blocks": 4.900438493352056e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010459984338376672}
|
| 248 |
+
{"avg_context_frames": 14.4875, "avg_micro_time_s": 1.6559243969750241, "epoch": 27, "global_step": 1740, "grad_norm_max": 0.006134033203125, "hidden_mse": 0.011353890295140445, "loss": 0.011533448891714216, "lr_blocks": 4.549673247541875e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010398059844737873}
|
| 249 |
+
{"avg_context_frames": 14.481250000000001, "avg_micro_time_s": 1.6521249546349281, "epoch": 27, "global_step": 1750, "grad_norm_max": 0.005340576171875, "hidden_mse": 0.01085884579224512, "loss": 0.010458897368516774, "lr_blocks": 4.211333667247125e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009373012726427987}
|
| 250 |
+
{"avg_context_frames": 14.253125, "avg_micro_time_s": 1.6516157042235138, "epoch": 28, "global_step": 1760, "grad_norm_max": 0.00897216796875, "hidden_mse": 0.01099745903047733, "loss": 0.010915642289910467, "lr_blocks": 3.885512251130763e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009815896424697712}
|
| 251 |
+
{"avg_context_frames": 14.40625, "avg_micro_time_s": 1.644893983801012, "epoch": 28, "global_step": 1770, "grad_norm_max": 0.00506591796875, "hidden_mse": 0.011815862666117028, "loss": 0.011678584961919114, "lr_blocks": 3.572298075514652e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010496998694725335}
|
| 252 |
+
{"avg_context_frames": 14.200000000000001, "avg_micro_time_s": 1.6430497726985778, "epoch": 28, "global_step": 1780, "grad_norm_max": 0.005523681640625, "hidden_mse": 0.011236432031728328, "loss": 0.01140632214373909, "lr_blocks": 3.271776770026963e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.0102826789021492}
|
| 253 |
+
{"avg_context_frames": 14.068750000000001, "avg_micro_time_s": 1.6447425782131178, "epoch": 28, "global_step": 1790, "grad_norm_max": 0.006195068359375, "hidden_mse": 0.011388846929185093, "loss": 0.011349822208285332, "lr_blocks": 2.984030494191942e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010210937494412065}
|
| 254 |
+
{"avg_context_frames": 14.421875, "avg_micro_time_s": 1.6483115719613126, "epoch": 28, "global_step": 1800, "grad_norm_max": 0.006622314453125, "hidden_mse": 0.011306738364510238, "loss": 0.01128948276746087, "lr_blocks": 2.7091379149682683e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010158808931009844}
|
| 255 |
+
{"avg_context_frames": 14.34375, "avg_micro_time_s": 1.6450169983883827, "epoch": 28, "global_step": 1810, "grad_norm_max": 0.005523681640625, "hidden_mse": 0.01131106594693847, "loss": 0.011214539268985392, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01008343268185854}
|
| 256 |
+
{"avg_context_frames": 14.26875, "avg_micro_time_s": 1.6483923100764515, "epoch": 28, "global_step": 1820, "grad_norm_max": 0.006683349609375, "hidden_mse": 0.010966053180163728, "loss": 0.010649461433058605, "lr_blocks": 2.198210923282118e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009552856074878947}
|
| 257 |
+
{"avg_context_frames": 14.231250000000001, "avg_micro_time_s": 1.6480346050742811, "epoch": 29, "global_step": 1830, "grad_norm_max": 0.005767822265625, "hidden_mse": 0.011051499308086932, "loss": 0.01099624038906768, "lr_blocks": 1.962316193157593e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009891090454766528}
|
| 258 |
+
{"avg_context_frames": 14.65625, "avg_micro_time_s": 1.6622220165001638, "epoch": 29, "global_step": 1840, "grad_norm_max": 0.005279541015625, "hidden_mse": 0.01025027733994648, "loss": 0.010120689944596962, "lr_blocks": 1.7395544861325718e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009095662174513564}
|
| 259 |
+
{"avg_context_frames": 14.4, "avg_micro_time_s": 1.6728738018879086, "epoch": 29, "global_step": 1850, "grad_norm_max": 0.00616455078125, "hidden_mse": 0.011631351051619276, "loss": 0.011471145966788755, "lr_blocks": 1.5299867030334815e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010308010876178742}
|
| 260 |
+
{"avg_context_frames": 14.040625, "avg_micro_time_s": 1.6607555011127262, "epoch": 29, "global_step": 1860, "grad_norm_max": 0.005706787109375, "hidden_mse": 0.011891721456777304, "loss": 0.011896548618096858, "lr_blocks": 1.333670137599713e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.01070737650152296}
|
| 261 |
+
{"avg_context_frames": 14.4, "avg_micro_time_s": 1.666131774499445, "epoch": 29, "global_step": 1870, "grad_norm_max": 0.00555419921875, "hidden_mse": 0.011788329185219482, "loss": 0.011521958839148284, "lr_blocks": 1.1506584608200366e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010343125910731034}
|
| 262 |
+
{"avg_context_frames": 14.0875, "avg_micro_time_s": 1.6488830267502637, "epoch": 29, "global_step": 1880, "grad_norm_max": 0.00457763671875, "hidden_mse": 0.011259010009234773, "loss": 0.01129114049836062, "lr_blocks": 9.810017062595322e-08, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010165239439811558}
|
| 263 |
+
{"avg_context_frames": 14.334375000000001, "avg_micro_time_s": 1.658405292975658, "epoch": 30, "global_step": 1890, "grad_norm_max": 0.00701904296875, "hidden_mse": 0.01104553140467033, "loss": 0.011151871737092735, "lr_blocks": 8.247462563808816e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010047318600118161}
|
| 264 |
+
{"avg_context_frames": 14.3375, "avg_micro_time_s": 1.6700679287012463, "epoch": 30, "global_step": 1900, "grad_norm_max": 0.00677490234375, "hidden_mse": 0.011436432937625796, "loss": 0.011454507318558172, "lr_blocks": 6.819348298638839e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010310864035272971}
|
| 265 |
+
{"avg_context_frames": 14.175, "avg_micro_time_s": 1.6541614500634751, "epoch": 30, "global_step": 1910, "grad_norm_max": 0.0052490234375, "hidden_mse": 0.011174249934265391, "loss": 0.010894450556952507, "lr_blocks": 5.526064699265754e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009777025505900384}
|
| 266 |
+
{"avg_context_frames": 14.200000000000001, "avg_micro_time_s": 1.6462076358369815, "epoch": 30, "global_step": 1920, "grad_norm_max": 0.0081787109375, "hidden_mse": 0.011725715646753088, "loss": 0.011740783043205739, "lr_blocks": 4.367965336512403e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010568211472127588}
|
| 267 |
+
{"avg_context_frames": 14.15, "avg_micro_time_s": 1.6475436234864902, "epoch": 30, "global_step": 1930, "grad_norm_max": 0.00604248046875, "hidden_mse": 0.011032538558356465, "loss": 0.010990633897017689, "lr_blocks": 3.345366823180929e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009887380053987727}
|
| 268 |
+
{"avg_context_frames": 14.137500000000001, "avg_micro_time_s": 1.6395516200249405, "epoch": 30, "global_step": 1940, "grad_norm_max": 0.004638671875, "hidden_mse": 0.011372241860954091, "loss": 0.011327299562981353, "lr_blocks": 2.4585487274942922e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010190075432183221}
|
| 269 |
+
{"avg_context_frames": 14.5875, "avg_micro_time_s": 1.6594994173243323, "epoch": 30, "global_step": 1950, "grad_norm_max": 0.006378173828125, "hidden_mse": 0.010940364468842745, "loss": 0.010809329786570744, "lr_blocks": 1.7077534966650767e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009715293342014776}
|
| 270 |
+
{"avg_context_frames": 14.481250000000001, "avg_micro_time_s": 1.6560149330744025, "epoch": 31, "global_step": 1960, "grad_norm_max": 0.00433349609375, "hidden_mse": 0.011649387062061579, "loss": 0.011777725949650632, "lr_blocks": 1.0931863906127327e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010612787236459554}
|
| 271 |
+
{"avg_context_frames": 14.256250000000001, "avg_micro_time_s": 1.6530842161268084, "epoch": 31, "global_step": 1970, "grad_norm_max": 0.007781982421875, "hidden_mse": 0.01121381939155981, "loss": 0.010996508109383286, "lr_blocks": 6.150154258476315e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009875126113183797}
|
| 272 |
+
{"avg_context_frames": 14.396875000000001, "avg_micro_time_s": 1.6463287756854699, "epoch": 31, "global_step": 1980, "grad_norm_max": 0.00567626953125, "hidden_mse": 0.010809205658733846, "loss": 0.010781579179456458, "lr_blocks": 2.7337132953697555e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009700658690417186}
|
| 273 |
+
{"avg_context_frames": 14.309375000000001, "avg_micro_time_s": 1.6505390620754044, "epoch": 31, "global_step": 1990, "grad_norm_max": 0.005279541015625, "hidden_mse": 0.011214246426243336, "loss": 0.01118017845437862, "lr_blocks": 6.834750376549793e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.010058753815246747}
|
| 274 |
+
{"avg_context_frames": 14.203125, "avg_micro_time_s": 1.6564274414478861, "epoch": 31, "global_step": 2000, "grad_norm_max": 0.00537109375, "hidden_mse": 0.011120558704715223, "loss": 0.011053164611803369, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 31.181796550750732, "velocity_mse": 0.009941108722705394}
|
| 275 |
+
swanlab: 📁 View project at
|
| 276 |
+
https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-DisCa
|
| 277 |
+
swanlab: 🚀 View run predictor-disca-0-53-b16x8-2000steps at
|
| 278 |
+
https://swanlab.cn/@ZhuoChen/HY-WorldPlay-Predictor-DisCa/runs/8mjozf71
|
| 279 |
+
swanlab: Upload complete: 25337 records
|
checkpoints/predictor_disca_blocks0-53_bf16_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/training_latest.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:975c4d8ffdafc2d22e7b5e7ea69025287f61b6d17320515f58dd3e9014e1d96b
|
| 3 |
+
size 1359629862
|
checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/launcher.nohup.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/predictor_step_00100.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:74bff71310408ec86e7f81030497ee3aeb2e871c663912533d34f9d4b390bc36
|
| 3 |
+
size 885425804
|
checkpoints/predictor_v4_atc-full_transport-global-chunk_blocks1-52_fulltrain100_fp32master_4gpu_bs8x2_flr1e-4_blr1e-5_steps2000/predictor_step_00200.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eea4729a38c187ba50a034aedc6256c9aae2b9afef8fbe72300f1f2817b6579d
|
| 3 |
+
size 885425804
|