EXP_027_davidnet-e2e_s42: checkpoint step 1500

experiments/EXP_027_davidnet-e2e_s42/checkpoints/LATEST.json

@@ -1,10 +1,10 @@
 {
-  "step": 1000,
-  "dir": "checkpoints/step-0001000",
-  "updated_at": "2026-09-27T23:37:34Z",
+  "step": 1500,
+  "dir": "checkpoints/step-0001500",
+  "updated_at": "2026-09-27T23:50:36Z",
   "config_hash": "98874fcb3c32",
   "history": [
-    "checkpoints/step-0000500",
-    "checkpoints/step-0001000"
+    "checkpoints/step-0001000",
+    "checkpoints/step-0001500"
   ]
 }

experiments/EXP_027_davidnet-e2e_s42/checkpoints/{step-0000500 → step-0001500}/meta.json

@@ -1,13 +1,13 @@
 {
   "config_hash": "98874fcb3c32",
-  "created_at": "2026-09-27T23:24:17Z",
-  "epoch": 0,
+  "created_at": "2026-09-27T23:50:30Z",
+  "epoch": 2,
   "files": {
     "training_state.pt": {
-      "bytes": 1952898037,
-      "sha256": "ab63218a041cdd906b4b5dff2897584055db1473833a89520630f64fb8f17718"
+      "bytes": 1952899509,
+      "sha256": "1adea960e4f3bf0b65b8b51e1567c9105693fd52fc22ff66badedf646860aa13"
     }
   },
   "reason": "interval",
-  "step": 500
+  "step": 1500
 }

experiments/EXP_027_davidnet-e2e_s42/checkpoints/{step-0000500 → step-0001500}/training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab63218a041cdd906b4b5dff2897584055db1473833a89520630f64fb8f17718
-size 1952898037
+oid sha256:1adea960e4f3bf0b65b8b51e1567c9105693fd52fc22ff66badedf646860aa13
+size 1952899509

experiments/EXP_027_davidnet-e2e_s42/logs/pipeline.jsonl

@@ -256,3 +256,7 @@
 {"t": "2026-09-27T23:26:35Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_027 epoch 1/6 (epoch=1, step=543, lr=2.962e-05, train/v=0.1318, train/a=0.02023, train/quad=0.404, train/loc=0.374, train/sync=1.502, train/dis=0.009544, train/total=0.6921, val/v_loss=0.09265, val/a_loss=0.007117, val/quad_loss=0.3433, val/loc_loss=0.3515, val/sync_loss=0.4117, val/dis_loss=0.01272, val/loss=0.4896, val/video_auc=0.9752, val/video_f1@0.5=0.9425, val/audio_auc=0.9999, val/audio_f1@0.5=0.9878, val/clip_auc=0.9768, val/clip_f1@0.5=0.9715, val/mean_auc=0.9876, val/quad_macro_f1=0.8921, epoch_seconds=853.2, val_seconds=63.9, best=0.9876, improved=True)", "epoch": 1, "step": 543, "lr": 2.961518973100432e-05, "train/v": 0.13178406729245623, "train/a": 0.020228297356042185, "train/quad": 0.40397688164802165, "train/loc": 0.3739864173104732, "train/sync": 1.5016832600860526, "train/dis": 0.009544097285721585, "train/total": 0.6921167522869832, "val/v_loss": 0.09264947623523767, "val/a_loss": 0.007117237655053707, "val/quad_loss": 0.34331799340680635, "val/loc_loss": 0.3515368688401605, "val/sync_loss": 0.4117398518782396, "val/dis_loss": 0.012717507349757047, "val/loss": 0.4896398889348627, "val/video_auc": 0.9752190476190475, "val/video_f1@0.5": 0.9424657534246575, "val/audio_auc": 0.9999060385868204, "val/audio_f1@0.5": 0.9878345498783455, "val/clip_auc": 0.9767576243980738, "val/clip_f1@0.5": 0.9714937286202965, "val/mean_auc": 0.987562543102934, "val/quad_macro_f1": 0.892055570439907, "epoch_seconds": 853.2, "val_seconds": 63.9, "best": 0.987562543102934, "improved": true}
 {"t": "2026-09-27T23:26:35Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.987562543102934"}
 {"t": "2026-09-27T23:37:33Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1000 (interval) (step=1000, epoch=1, mb=1953)", "step": 1000, "epoch": 1, "mb": 1952.9}
+{"t": "2026-09-27T23:38:04Z", "level": "INFO", "event": "checkpoint_uploaded", "msg": "[checkpoint_uploaded] step 1000 verified on HF (step=1000, seconds=30.1, with_best=True)", "step": 1000, "seconds": 30.1, "with_best": true}
+{"t": "2026-09-27T23:40:41Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_027 epoch 2/6 (epoch=2, step=1086, lr=2.54e-05, train/v=0.02697, train/a=0.006159, train/quad=0.1035, train/loc=0.1094, train/sync=1.349, train/dis=0.01001, train/total=0.2756, val/v_loss=0.1323, val/a_loss=0.04133, val/quad_loss=0.5597, val/loc_loss=0.5378, val/sync_loss=0.3605, val/dis_loss=0.009755, val/loss=0.7594, val/video_auc=0.974, val/video_f1@0.5=0.9495, val/audio_auc=0.9996, val/audio_f1@0.5=0.9703, val/clip_auc=0.9741, val/clip_f1@0.5=0.9672, val/mean_auc=0.9868, val/quad_macro_f1=0.8828, epoch_seconds=846, val_seconds=62.9, best=0.9876, improved=False)", "epoch": 2, "step": 1086, "lr": 2.5403079081282112e-05, "train/v": 0.026972717428599323, "train/a": 0.006158842493463612, "train/quad": 0.10354597383072592, "train/loc": 0.10941284399725215, "train/sync": 1.3494810869241507, "train/dis": 0.010011935120463049, "train/total": 0.2755602742860732, "val/v_loss": 0.13234402781976173, "val/a_loss": 0.04132815902884536, "val/quad_loss": 0.559697407377014, "val/loc_loss": 0.5377941766805844, "val/sync_loss": 0.36053566565880407, "val/dis_loss": 0.009755457296537665, "val/loss": 0.7594470969266974, "val/video_auc": 0.974047619047619, "val/video_f1@0.5": 0.949468085106383, "val/audio_auc": 0.9995615134051616, "val/audio_f1@0.5": 0.9702970297029703, "val/clip_auc": 0.9741091492776887, "val/clip_f1@0.5": 0.9671574178935447, "val/mean_auc": 0.9868045662263903, "val/quad_macro_f1": 0.8828272140766591, "epoch_seconds": 846.0, "val_seconds": 62.9, "best": 0.987562543102934, "improved": false}
+{"t": "2026-09-27T23:40:41Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9868045662263903"}
+{"t": "2026-09-27T23:50:36Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1500 (interval) (step=1500, epoch=2, mb=1953)", "step": 1500, "epoch": 2, "mb": 1952.9}

experiments/EXP_027_davidnet-e2e_s42/metrics/training_history.csv

@@ -1,2 +1,3 @@
 epoch,step,lr,train/v,train/a,train/quad,train/loc,train/sync,train/dis,train/total,val/v_loss,val/a_loss,val/quad_loss,val/loc_loss,val/sync_loss,val/dis_loss,val/loss,val/video_auc,val/video_f1@0.5,val/audio_auc,val/audio_f1@0.5,val/clip_auc,val/clip_f1@0.5,val/mean_auc,val/quad_macro_f1,epoch_seconds,val_seconds
 1,543,2.961518973100432e-05,0.13178406729245623,0.020228297356042185,0.40397688164802165,0.3739864173104732,1.5016832600860526,0.009544097285721585,0.6921167522869832,0.09264947623523767,0.007117237655053707,0.34331799340680635,0.3515368688401605,0.4117398518782396,0.012717507349757047,0.4896398889348627,0.9752190476190475,0.9424657534246575,0.9999060385868204,0.9878345498783455,0.9767576243980738,0.9714937286202965,0.987562543102934,0.892055570439907,853.2,63.9
+2,1086,2.5403079081282112e-05,0.026972717428599323,0.006158842493463612,0.10354597383072592,0.10941284399725215,1.3494810869241507,0.010011935120463049,0.2755602742860732,0.13234402781976173,0.04132815902884536,0.559697407377014,0.5377941766805844,0.36053566565880407,0.009755457296537665,0.7594470969266974,0.974047619047619,0.949468085106383,0.9995615134051616,0.9702970297029703,0.9741091492776887,0.9671574178935447,0.9868045662263903,0.8828272140766591,846.0,62.9