EXP_005_davidnet_s42: checkpoint step 1500

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/LATEST.json

@@ -1,10 +1,10 @@
 {
-  "step": 1000,
-  "dir": "checkpoints/step-0001000",
-  "updated_at": "2026-10-05T02:02:14Z",
+  "step": 1500,
+  "dir": "checkpoints/step-0001500",
+  "updated_at": "2026-10-05T02:05:28Z",
   "config_hash": "504b19d07f0d",
   "history": [
-    "checkpoints/step-0000500",
-    "checkpoints/step-0001000"
+    "checkpoints/step-0001000",
+    "checkpoints/step-0001500"
   ]
 }

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/{step-0000500 → step-0001500}/meta.json

@@ -1,13 +1,13 @@
 {
   "config_hash": "504b19d07f0d",
-  "created_at": "2026-10-05T01:58:53Z",
-  "epoch": 3,
+  "created_at": "2026-10-05T02:05:25Z",
+  "epoch": 11,
   "files": {
     "training_state.pt": {
-      "bytes": 1003381011,
-      "sha256": "5f58e5d6dc61e3c2568cae07683551bdccc1f78e7a4dfd5711c5b39c88ed2ad5"
+      "bytes": 1003386515,
+      "sha256": "b38cd0aba7f680cc6fbfaaef9dd05e17e194aaf7caf1d5b09814006fe65659ca"
     }
   },
   "reason": "interval",
-  "step": 500
+  "step": 1500
 }

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/{step-0000500 → step-0001500}/training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f58e5d6dc61e3c2568cae07683551bdccc1f78e7a4dfd5711c5b39c88ed2ad5
-size 1003381011
+oid sha256:b38cd0aba7f680cc6fbfaaef9dd05e17e194aaf7caf1d5b09814006fe65659ca
+size 1003386515

repro_check/experiments/EXP_005_davidnet_s42/logs/pipeline.jsonl

@@ -281,3 +281,13 @@
 {"t": "2026-10-05T02:01:53Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 7/30 (epoch=7, step=952, lr=9.125e-05, train/v=0.02142, train/a=0.003294, train/quad=0.08544, train/loc=0.08349, train/sync=3.889, train/dis=0.02521, train/total=0.5006, val/v_loss=0.1322, val/a_loss=0.006299, val/quad_loss=0.4696, val/loc_loss=0.4262, val/sync_loss=0.941, val/dis_loss=0.02581, val/loss=0.6831, val/video_auc=0.9814, val/video_f1@0.5=0.9439, val/audio_auc=0.9997, val/audio_f1@0.5=0.9904, val/clip_auc=0.9785, val/clip_f1@0.5=0.9634, val/mean_auc=0.9906, val/quad_macro_f1=0.9041, epoch_seconds=51.6, val_seconds=2, best=0.9906, improved=True)", "epoch": 7, "step": 952, "lr": 9.124646526761206e-05, "train/v": 0.021422295119384004, "train/a": 0.0032936235872164294, "train/quad": 0.0854361842639799, "train/loc": 0.08349221114322598, "train/sync": 3.8886290578281177, "train/dis": 0.025205880978747326, "train/total": 0.5005636171383017, "val/v_loss": 0.13222642118732134, "val/a_loss": 0.0062992111201714985, "val/quad_loss": 0.4696454405784607, "val/loc_loss": 0.42616500871049035, "val/sync_loss": 0.9410407808091905, "val/dis_loss": 0.025806931571827993, "val/loss": 0.68311564789878, "val/video_auc": 0.9814, "val/video_f1@0.5": 0.9439124487004104, "val/audio_auc": 0.9997416061137561, "val/audio_f1@0.5": 0.9903846153846154, "val/clip_auc": 0.9784751203852327, "val/clip_f1@0.5": 0.9633867276887872, "val/mean_auc": 0.9905708030568781, "val/quad_macro_f1": 0.9041083561224227, "epoch_seconds": 51.6, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": true}
 {"t": "2026-10-05T02:01:53Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9905708030568781"}
 {"t": "2026-10-05T02:02:14Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1000 (interval) (step=1000, epoch=7, mb=1003)", "step": 1000, "epoch": 7, "mb": 1003.4}
+{"t": "2026-10-05T02:02:28Z", "level": "INFO", "event": "checkpoint_uploaded", "msg": "[checkpoint_uploaded] step 1000 verified on HF (step=1000, seconds=13.9, with_best=True)", "step": 1000, "seconds": 13.9, "with_best": true}
+{"t": "2026-10-05T02:02:48Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 8/30 (epoch=8, step=1088, lr=8.791e-05, train/v=0.03041, train/a=0.001611, train/quad=0.1072, train/loc=0.1114, train/sync=3.796, train/dis=0.01996, train/total=0.5229, val/v_loss=0.1768, val/a_loss=0.02742, val/quad_loss=0.6912, val/loc_loss=0.6359, val/sync_loss=1.062, val/dis_loss=0.01476, val/loss=0.9754, val/video_auc=0.979, val/video_f1@0.5=0.9374, val/audio_auc=0.9988, val/audio_f1@0.5=0.9858, val/clip_auc=0.9743, val/clip_f1@0.5=0.9622, val/mean_auc=0.9889, val/quad_macro_f1=0.8918, epoch_seconds=55.5, val_seconds=2, best=0.9906, improved=False)", "epoch": 8, "step": 1088, "lr": 8.790721531620747e-05, "train/v": 0.03041003642626011, "train/a": 0.0016112352231616837, "train/quad": 0.10723242379283286, "train/loc": 0.11136850831954849, "train/sync": 3.7959871888160706, "train/dis": 0.019959901483482954, "train/total": 0.5229164537261514, "val/v_loss": 0.17680589637408653, "val/a_loss": 0.027423809881687095, "val/quad_loss": 0.6911837736972504, "val/loc_loss": 0.635944644610087, "val/sync_loss": 1.0616210831536188, "val/dis_loss": 0.014762035467558436, "val/loss": 0.9754322518905004, "val/video_auc": 0.979, "val/video_f1@0.5": 0.9374130737134909, "val/audio_auc": 0.9988098220997244, "val/audio_f1@0.5": 0.985781990521327, "val/clip_auc": 0.9742536115569823, "val/clip_f1@0.5": 0.9621993127147767, "val/mean_auc": 0.9889049110498622, "val/quad_macro_f1": 0.891759900960856, "epoch_seconds": 55.5, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": false}
+{"t": "2026-10-05T02:02:48Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9889049110498622"}
+{"t": "2026-10-05T02:03:40Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 9/30 (epoch=9, step=1224, lr=8.411e-05, train/v=0.01628, train/a=0.00377, train/quad=0.06331, train/loc=0.06784, train/sync=3.698, train/dis=0.027, train/total=0.4581, val/v_loss=0.06284, val/a_loss=0.002254, val/quad_loss=0.2239, val/loc_loss=0.2178, val/sync_loss=0.9212, val/dis_loss=0.01134, val/loss=0.3792, val/video_auc=0.98, val/video_f1@0.5=0.9549, val/audio_auc=1, val/audio_f1@0.5=0.9976, val/clip_auc=0.9828, val/clip_f1@0.5=0.9707, val/mean_auc=0.99, val/quad_macro_f1=0.9045, epoch_seconds=51.6, val_seconds=2, best=0.9906, improved=False)", "epoch": 9, "step": 1224, "lr": 8.411389244402088e-05, "train/v": 0.01627865637483945, "train/a": 0.003769871600463362, "train/quad": 0.06330716210119564, "train/loc": 0.06783826308845378, "train/sync": 3.697898103910334, "train/dis": 0.027000088997952202, "train/total": 0.45811106511117783, "val/v_loss": 0.06283980564007328, "val/a_loss": 0.0022539284816376595, "val/quad_loss": 0.22392648779269722, "val/loc_loss": 0.21778364520933893, "val/sync_loss": 0.9212481710645888, "val/dis_loss": 0.01134108550225695, "val/loss": 0.37920773981346023, "val/video_auc": 0.9800000000000001, "val/video_f1@0.5": 0.9549071618037135, "val/audio_auc": 0.9999530192934102, "val/audio_f1@0.5": 0.9975903614457832, "val/clip_auc": 0.9827608346709471, "val/clip_f1@0.5": 0.9707207207207207, "val/mean_auc": 0.9899765096467051, "val/quad_macro_f1": 0.9044558458067159, "epoch_seconds": 51.6, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": false}
+{"t": "2026-10-05T02:03:40Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9899765096467051"}
+{"t": "2026-10-05T02:04:31Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 10/30 (epoch=10, step=1360, lr=7.994e-05, train/v=0.01728, train/a=0.006979, train/quad=0.08085, train/loc=0.08245, train/sync=3.564, train/dis=0.02737, train/total=0.4651, val/v_loss=0.07465, val/a_loss=0.02453, val/quad_loss=0.334, val/loc_loss=0.3291, val/sync_loss=0.878, val/dis_loss=0.02012, val/loss=0.5206, val/video_auc=0.9756, val/video_f1@0.5=0.9536, val/audio_auc=0.9997, val/audio_f1@0.5=0.9835, val/clip_auc=0.98, val/clip_f1@0.5=0.9719, val/mean_auc=0.9877, val/quad_macro_f1=0.9002, epoch_seconds=51.5, val_seconds=2, best=0.9906, improved=False)", "epoch": 10, "step": 1360, "lr": 7.994480298371163e-05, "train/v": 0.01728180047361735, "train/a": 0.006978785372827106, "train/quad": 0.08084832882227869, "train/loc": 0.08244854963285274, "train/sync": 3.564415174372056, "train/dis": 0.02736562627764857, "train/total": 0.46508711313500123, "val/v_loss": 0.07464678968406385, "val/a_loss": 0.02452731575168501, "val/quad_loss": 0.3340199569033252, "val/loc_loss": 0.329137915538417, "val/sync_loss": 0.8780298762851291, "val/dis_loss": 0.020119365511669055, "val/loss": 0.5205679693155818, "val/video_auc": 0.9756476190476191, "val/video_f1@0.5": 0.9536423841059603, "val/audio_auc": 0.9996711350538712, "val/audio_f1@0.5": 0.983451536643026, "val/clip_auc": 0.98, "val/clip_f1@0.5": 0.9718785151856018, "val/mean_auc": 0.9876593770507451, "val/quad_macro_f1": 0.9001765815853837, "epoch_seconds": 51.5, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": false}
+{"t": "2026-10-05T02:04:31Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9876593770507451"}
+{"t": "2026-10-05T02:05:23Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 11/30 (epoch=11, step=1496, lr=7.539e-05, train/v=0.01133, train/a=0.001863, train/quad=0.04027, train/loc=0.04381, train/sync=3.475, train/dis=0.01851, train/total=0.4046, val/v_loss=0.07528, val/a_loss=0.002414, val/quad_loss=0.2887, val/loc_loss=0.2614, val/sync_loss=1.021, val/dis_loss=0.01046, val/loss=0.4559, val/video_auc=0.9757, val/video_f1@0.5=0.9465, val/audio_auc=1, val/audio_f1@0.5=0.9928, val/clip_auc=0.9842, val/clip_f1@0.5=0.9704, val/mean_auc=0.9878, val/quad_macro_f1=0.9033, epoch_seconds=51.5, val_seconds=2, best=0.9906, improved=False)", "epoch": 11, "step": 1496, "lr": 7.538885275878594e-05, "train/v": 0.011332837240950218, "train/a": 0.0018626836741692816, "train/quad": 0.040273989742512215, "train/loc": 0.04380577014714671, "train/sync": 3.474968712119495, "train/dis": 0.018506204997024992, "train/total": 0.4045829001594992, "val/v_loss": 0.075275970933338, "val/a_loss": 0.0024137285663022746, "val/quad_loss": 0.28867936444779235, "val/loc_loss": 0.26137485976020497, "val/sync_loss": 1.021100680033366, "val/dis_loss": 0.010457693495684199, "val/loss": 0.45587265657054055, "val/video_auc": 0.9756952380952382, "val/video_f1@0.5": 0.946524064171123, "val/audio_auc": 1.0, "val/audio_f1@0.5": 0.9928400954653938, "val/clip_auc": 0.9842054574638845, "val/clip_f1@0.5": 0.9703872437357631, "val/mean_auc": 0.9878476190476191, "val/quad_macro_f1": 0.9033292277833762, "epoch_seconds": 51.5, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": false}
+{"t": "2026-10-05T02:05:23Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9878476190476191"}
+{"t": "2026-10-05T02:05:28Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1500 (interval) (step=1500, epoch=11, mb=1003)", "step": 1500, "epoch": 11, "mb": 1003.4}

repro_check/experiments/EXP_005_davidnet_s42/metrics/training_history.csv

@@ -6,3 +6,7 @@ epoch,step,lr,train/v,train/a,train/quad,train/loc,train/sync,train/dis,train/to
 5,680,9.640661438195972e-05,0.039419552112710386,0.0034818905216712557,0.1486379991778556,0.1541441438089618,3.872383061577292,0.019989624616059968,0.5835297861519981,0.13556796384768355,0.015706277119332097,0.5577797297802236,0.5102543276217248,1.0061896642049153,0.047482126909825534,0.7906584607230293,0.9755142857142857,0.94750656167979,0.9993422701077423,0.985781990521327,0.9747191011235955,0.9646017699115044,0.987428277911014,0.8903043226031744,51.6,2.0
 6,816,9.409110832729244e-05,0.035125741569037715,0.004922062038571419,0.1348276804543703,0.13791961004437112,3.911714923732421,0.02554565987563418,0.5701475160963395,0.09950710232887003,0.008299460592954548,0.37559549551871085,0.3852198181880845,0.9967135853237576,0.02103537248654498,0.589989110827446,0.9654666666666666,0.9389920424403183,0.9997807567025807,0.9856459330143541,0.9704173354735154,0.9616252821670429,0.9826237116846237,0.8851123225949468,51.5,2.0
 7,952,9.124646526761206e-05,0.021422295119384004,0.0032936235872164294,0.0854361842639799,0.08349221114322598,3.8886290578281177,0.025205880978747326,0.5005636171383017,0.13222642118732134,0.0062992111201714985,0.4696454405784607,0.42616500871049035,0.9410407808091905,0.025806931571827993,0.68311564789878,0.9814,0.9439124487004104,0.9997416061137561,0.9903846153846154,0.9784751203852327,0.9633867276887872,0.9905708030568781,0.9041083561224227,51.6,2.0
+8,1088,8.790721531620747e-05,0.03041003642626011,0.0016112352231616837,0.10723242379283286,0.11136850831954849,3.7959871888160706,0.019959901483482954,0.5229164537261514,0.17680589637408653,0.027423809881687095,0.6911837736972504,0.635944644610087,1.0616210831536188,0.014762035467558436,0.9754322518905004,0.979,0.9374130737134909,0.9988098220997244,0.985781990521327,0.9742536115569823,0.9621993127147767,0.9889049110498622,0.891759900960856,55.5,2.0
+9,1224,8.411389244402088e-05,0.01627865637483945,0.003769871600463362,0.06330716210119564,0.06783826308845378,3.697898103910334,0.027000088997952202,0.45811106511117783,0.06283980564007328,0.0022539284816376595,0.22392648779269722,0.21778364520933893,0.9212481710645888,0.01134108550225695,0.37920773981346023,0.9800000000000001,0.9549071618037135,0.9999530192934102,0.9975903614457832,0.9827608346709471,0.9707207207207207,0.9899765096467051,0.9044558458067159,51.6,2.0
+10,1360,7.994480298371163e-05,0.01728180047361735,0.006978785372827106,0.08084832882227869,0.08244854963285274,3.564415174372056,0.02736562627764857,0.46508711313500123,0.07464678968406385,0.02452731575168501,0.3340199569033252,0.329137915538417,0.8780298762851291,0.020119365511669055,0.5205679693155818,0.9756476190476191,0.9536423841059603,0.9996711350538712,0.983451536643026,0.98,0.9718785151856018,0.9876593770507451,0.9001765815853837,51.5,2.0
+11,1496,7.538885275878594e-05,0.011332837240950218,0.0018626836741692816,0.040273989742512215,0.04380577014714671,3.474968712119495,0.018506204997024992,0.4045829001594992,0.075275970933338,0.0024137285663022746,0.28867936444779235,0.26137485976020497,1.021100680033366,0.010457693495684199,0.45587265657054055,0.9756952380952382,0.946524064171123,1.0,0.9928400954653938,0.9842054574638845,0.9703872437357631,0.9878476190476191,0.9033292277833762,51.5,2.0