EXP_016_davidnet-single_task_s42: checkpoint step 1904

experiments/EXP_016_davidnet-single_task_s42/checkpoints/LATEST.json

@@ -1,10 +1,10 @@
 {
-  "step": 1500,
-  "dir": "checkpoints/step-0001500",
-  "updated_at": "2026-09-27T22:14:46Z",
+  "step": 1904,
+  "dir": "checkpoints/step-0001904",
+  "updated_at": "2026-09-27T22:17:26Z",
   "config_hash": "1129548a714e",
   "history": [
-    "checkpoints/step-0001000",
-    "checkpoints/step-0001500"
+    "checkpoints/step-0001500",
+    "checkpoints/step-0001904"
   ]
 }

experiments/EXP_016_davidnet-single_task_s42/checkpoints/step-0001000/meta.json

@@ -1,13 +0,0 @@
-{
-  "config_hash": "1129548a714e",
-  "created_at": "2026-09-27T22:11:23Z",
-  "epoch": 7,
-  "files": {
-    "training_state.pt": {
-      "bytes": 1003383763,
-      "sha256": "e5be296cfa81e48ec96344336035260e2d029b2a6996f05ddbf44bc7b7b94c43"
-    }
-  },
-  "reason": "interval",
-  "step": 1000
-}

experiments/EXP_016_davidnet-single_task_s42/checkpoints/step-0001904/meta.json

@@ -0,0 +1,13 @@
+{
+  "config_hash": "1129548a714e",
+  "created_at": "2026-09-27T22:17:23Z",
+  "epoch": 14,
+  "files": {
+    "training_state.pt": {
+      "bytes": 1003388435,
+      "sha256": "f515bd302c389b262a80f6341e45a0664a7fa5344ca938cf3f1976568d35a5e0"
+    }
+  },
+  "reason": "completed",
+  "step": 1904
+}

experiments/EXP_016_davidnet-single_task_s42/checkpoints/{step-0001000 → step-0001904}/training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5be296cfa81e48ec96344336035260e2d029b2a6996f05ddbf44bc7b7b94c43
-size 1003383763
+oid sha256:f515bd302c389b262a80f6341e45a0664a7fa5344ca938cf3f1976568d35a5e0
+size 1003388435

experiments/EXP_016_davidnet-single_task_s42/logs/pipeline.jsonl

@@ -238,3 +238,12 @@
 {"t": "2026-09-27T22:14:41Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_016 epoch 11/30 (epoch=11, step=1496, lr=7.528e-05, train/v=0.007233, train/a=0.0003815, train/quad=1.414, train/loc=1.103, train/sync=5.681, train/dis=0.0527, train/total=0.007614, val/v_loss=0.2247, val/a_loss=0.008093, val/quad_loss=1.409, val/loc_loss=1.151, val/sync_loss=1.531, val/dis_loss=0.05307, val/loss=0.2328, val/video_auc=0.9834, val/video_f1@0.5=0.9444, val/audio_auc=0.9999, val/audio_f1@0.5=0.9881, val/clip_auc=0.9874, val/clip_f1@0.5=0.963, val/mean_auc=0.9917, val/quad_macro_f1=0.0519, epoch_seconds=53.4, val_seconds=2.1, best=0.9942, improved=False)", "epoch": 11, "step": 1496, "lr": 7.528473763599666e-05, "train/v": 0.007232575288392528, "train/a": 0.0003815251758948414, "train/quad": 1.4144308260258507, "train/loc": 1.1030777342179243, "train/sync": 5.680546262684991, "train/dis": 0.052701691165566444, "train/total": 0.0076141004581800455, "val/v_loss": 0.2246906157169077, "val/a_loss": 0.008093261082125183, "val/quad_loss": 1.4091990258958604, "val/loc_loss": 1.1510257853402033, "val/sync_loss": 1.5310137006971571, "val/dis_loss": 0.05306866723630163, "val/loss": 0.2327838714958893, "val/video_auc": 0.9834190476190476, "val/video_f1@0.5": 0.9444444444444444, "val/audio_auc": 0.9998903783512905, "val/audio_f1@0.5": 0.9881235154394299, "val/clip_auc": 0.9873675762439806, "val/clip_f1@0.5": 0.9629629629629629, "val/mean_auc": 0.991654712985169, "val/quad_macro_f1": 0.0518958759291688, "epoch_seconds": 53.4, "val_seconds": 2.1, "best": 0.9942238095238094, "improved": false}
 {"t": "2026-09-27T22:14:41Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.991654712985169"}
 {"t": "2026-09-27T22:14:46Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1500 (interval) (step=1500, epoch=11, mb=1003)", "step": 1500, "epoch": 11, "mb": 1003.4}
+{"t": "2026-09-27T22:15:05Z", "level": "INFO", "event": "checkpoint_uploaded", "msg": "[checkpoint_uploaded] step 1500 verified on HF (step=1500, seconds=19.1, with_best=True)", "step": 1500, "seconds": 19.1, "with_best": true}
+{"t": "2026-09-27T22:15:38Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_016 epoch 12/30 (epoch=12, step=1632, lr=7.042e-05, train/v=0.005222, train/a=0.0008345, train/quad=1.409, train/loc=1.093, train/sync=5.558, train/dis=0.05543, train/total=0.006057, val/v_loss=0.1254, val/a_loss=0.04173, val/quad_loss=1.412, val/loc_loss=1.104, val/sync_loss=1.532, val/dis_loss=0.06478, val/loss=0.1671, val/video_auc=0.9858, val/video_f1@0.5=0.9576, val/audio_auc=0.9996, val/audio_f1@0.5=0.9765, val/clip_auc=0.9855, val/clip_f1@0.5=0.9753, val/mean_auc=0.9927, val/quad_macro_f1=0.03951, epoch_seconds=56.7, val_seconds=2.1, best=0.9942, improved=False)", "epoch": 12, "step": 1632, "lr": 7.042065868884853e-05, "train/v": 0.005222499565046318, "train/a": 0.0008345082181887499, "train/quad": 1.4094273036017138, "train/loc": 1.0930297357194565, "train/sync": 5.557975872474558, "train/dis": 0.05542819930569214, "train/total": 0.006057007801620571, "val/v_loss": 0.12541274995439583, "val/a_loss": 0.04172586908085426, "val/quad_loss": 1.4119592640135024, "val/loc_loss": 1.1040326025750902, "val/sync_loss": 1.5322035683525934, "val/dis_loss": 0.0647807696627246, "val/loss": 0.1671386216249731, "val/video_auc": 0.9857619047619047, "val/video_f1@0.5": 0.9575596816976127, "val/audio_auc": 0.9995771736406915, "val/audio_f1@0.5": 0.9765258215962441, "val/clip_auc": 0.9855377207062601, "val/clip_f1@0.5": 0.9752808988764045, "val/mean_auc": 0.9926695392012981, "val/quad_macro_f1": 0.039510353300770136, "epoch_seconds": 56.7, "val_seconds": 2.1, "best": 0.9942238095238094, "improved": false}
+{"t": "2026-09-27T22:15:38Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9926695392012981"}
+{"t": "2026-09-27T22:16:30Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_016 epoch 13/30 (epoch=13, step=1768, lr=6.531e-05, train/v=0.005438, train/a=0.004532, train/quad=1.412, train/loc=1.09, train/sync=5.6, train/dis=0.05634, train/total=0.00997, val/v_loss=0.1426, val/a_loss=0.03777, val/quad_loss=1.414, val/loc_loss=1.159, val/sync_loss=1.532, val/dis_loss=0.05744, val/loss=0.1804, val/video_auc=0.9862, val/video_f1@0.5=0.937, val/audio_auc=0.9999, val/audio_f1@0.5=0.972, val/clip_auc=0.9842, val/clip_f1@0.5=0.9661, val/mean_auc=0.9931, val/quad_macro_f1=0.01724, epoch_seconds=52.2, val_seconds=2.1, best=0.9942, improved=False)", "epoch": 13, "step": 1768, "lr": 6.531476993045718e-05, "train/v": 0.005437697613374707, "train/a": 0.004531993194175123, "train/quad": 1.4121647520976908, "train/loc": 1.0902323775431688, "train/sync": 5.600347333094653, "train/dis": 0.05633586661561447, "train/total": 0.0099696908310387, "val/v_loss": 0.1426076578077805, "val/a_loss": 0.03777275383213742, "val/quad_loss": 1.4137798282835219, "val/loc_loss": 1.1592833995819092, "val/sync_loss": 1.5315563413831923, "val/dis_loss": 0.05744102390276061, "val/loss": 0.18038040897065205, "val/video_auc": 0.9862285714285713, "val/video_f1@0.5": 0.9370277078085643, "val/audio_auc": 0.9998747181157605, "val/audio_f1@0.5": 0.9719626168224299, "val/clip_auc": 0.9842215088282503, "val/clip_f1@0.5": 0.966120218579235, "val/mean_auc": 0.9930516447721659, "val/quad_macro_f1": 0.017241379310344827, "epoch_seconds": 52.2, "val_seconds": 2.1, "best": 0.9942238095238094, "improved": false}
+{"t": "2026-09-27T22:16:30Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9930516447721659"}
+{"t": "2026-09-27T22:17:22Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_016 epoch 14/30 (epoch=14, step=1904, lr=6.003e-05, train/v=0.006326, train/a=0.001149, train/quad=1.413, train/loc=1.063, train/sync=5.612, train/dis=0.05355, train/total=0.007475, val/v_loss=0.1501, val/a_loss=0.02281, val/quad_loss=1.411, val/loc_loss=1.103, val/sync_loss=1.532, val/dis_loss=0.05923, val/loss=0.173, val/video_auc=0.987, val/video_f1@0.5=0.9664, val/audio_auc=0.9999, val/audio_f1@0.5=0.9834, val/clip_auc=0.989, val/clip_f1@0.5=0.9774, val/mean_auc=0.9934, val/quad_macro_f1=0.1131, epoch_seconds=52.4, val_seconds=2.1, best=0.9942, improved=False)", "epoch": 14, "step": 1904, "lr": 6.0029049933729254e-05, "train/v": 0.0063262600619696865, "train/a": 0.0011491511773944214, "train/quad": 1.4133971657823114, "train/loc": 1.0634809855152578, "train/sync": 5.612042083459742, "train/dis": 0.053549830473082906, "train/total": 0.007475411272453665, "val/v_loss": 0.15014608152624634, "val/a_loss": 0.0228123365290887, "val/quad_loss": 1.4109607405132718, "val/loc_loss": 1.1031819317075942, "val/sync_loss": 1.531647629208035, "val/dis_loss": 0.059230458819203906, "val/loss": 0.1729584193478028, "val/video_auc": 0.9869809523809524, "val/video_f1@0.5": 0.9663526244952894, "val/audio_auc": 0.9998747181157606, "val/audio_f1@0.5": 0.9833729216152018, "val/clip_auc": 0.9890208667736757, "val/clip_f1@0.5": 0.9774266365688488, "val/mean_auc": 0.9934278352483565, "val/quad_macro_f1": 0.1131444736632401, "epoch_seconds": 52.4, "val_seconds": 2.1, "best": 0.9942238095238094, "improved": false}
+{"t": "2026-09-27T22:17:22Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9934278352483565"}
+{"t": "2026-09-27T22:17:22Z", "level": "INFO", "event": "early_stop", "msg": "[early_stop] early stopping: no improvement in 6 epochs"}
+{"t": "2026-09-27T22:17:26Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1904 (completed) (step=1904, epoch=14, mb=1003)", "step": 1904, "epoch": 14, "mb": 1003.4}

experiments/EXP_016_davidnet-single_task_s42/metrics/training_history.csv

@@ -10,3 +10,6 @@ epoch,step,lr,train/v,train/a,train/quad,train/loc,train/sync,train/dis,train/to
 9,1224,8.4054944728694e-05,0.010796608663761564,0.0022793618941785313,1.4393335694775862,1.0742951587719076,5.599842713159673,0.05186962300692411,0.013075970566098797,0.08799470848761201,0.013110305052476987,1.4299536413616605,1.1359986861546834,1.5303444862365723,0.05645850031740136,0.10110501432791352,0.9836666666666667,0.9591567852437418,0.9996711350538712,0.9833729216152018,0.9867094703049758,0.9732142857142857,0.9916689008602689,0.0024691358024691358,52.5,2.1
 10,1360,7.984796344253914e-05,0.008374033254821209,0.001358118930304098,1.416895882171743,1.1062755426939797,5.649644294205834,0.04940582611871993,0.009732152183406631,0.06670908669649281,0.024322445205699397,1.4204443163341947,1.1219863361782498,1.531095716688368,0.058785703447129994,0.0910315305150612,0.9832,0.95822454308094,0.9994049110498622,0.985781990521327,0.9874478330658105,0.9744160177975528,0.991302455524931,0.030412210740079593,52.5,2.1
 11,1496,7.528473763599666e-05,0.007232575288392528,0.0003815251758948414,1.4144308260258507,1.1030777342179243,5.680546262684991,0.052701691165566444,0.0076141004581800455,0.2246906157169077,0.008093261082125183,1.4091990258958604,1.1510257853402033,1.5310137006971571,0.05306866723630163,0.2327838714958893,0.9834190476190476,0.9444444444444444,0.9998903783512905,0.9881235154394299,0.9873675762439806,0.9629629629629629,0.991654712985169,0.0518958759291688,53.4,2.1
+12,1632,7.042065868884853e-05,0.005222499565046318,0.0008345082181887499,1.4094273036017138,1.0930297357194565,5.557975872474558,0.05542819930569214,0.006057007801620571,0.12541274995439583,0.04172586908085426,1.4119592640135024,1.1040326025750902,1.5322035683525934,0.0647807696627246,0.1671386216249731,0.9857619047619047,0.9575596816976127,0.9995771736406915,0.9765258215962441,0.9855377207062601,0.9752808988764045,0.9926695392012981,0.039510353300770136,56.7,2.1
+13,1768,6.531476993045718e-05,0.005437697613374707,0.004531993194175123,1.4121647520976908,1.0902323775431688,5.600347333094653,0.05633586661561447,0.0099696908310387,0.1426076578077805,0.03777275383213742,1.4137798282835219,1.1592833995819092,1.5315563413831923,0.05744102390276061,0.18038040897065205,0.9862285714285713,0.9370277078085643,0.9998747181157605,0.9719626168224299,0.9842215088282503,0.966120218579235,0.9930516447721659,0.017241379310344827,52.2,2.1
+14,1904,6.0029049933729254e-05,0.0063262600619696865,0.0011491511773944214,1.4133971657823114,1.0634809855152578,5.612042083459742,0.053549830473082906,0.007475411272453665,0.15014608152624634,0.0228123365290887,1.4109607405132718,1.1031819317075942,1.531647629208035,0.059230458819203906,0.1729584193478028,0.9869809523809524,0.9663526244952894,0.9998747181157606,0.9833729216152018,0.9890208667736757,0.9774266365688488,0.9934278352483565,0.1131444736632401,52.4,2.1