EXP_005_davidnet_s42: checkpoint step 2000

repro_check/experiments/EXP_005_davidnet_s42/best_model/meta.json

@@ -1,10 +1,10 @@
 {
   "config_hash": "504b19d07f0d",
-  "epoch": 7,
+  "epoch": 13,
   "exp_id": "EXP_005",
-  "metric": 0.9905708030568781,
+  "metric": 0.9914434954838864,
   "metric_name": "val/mean_auc",
-  "saved_at": "2026-10-05T02:01:52Z",
-  "sha256": "a4dddf8cbd5dc7a1bfddc3790ca23997d314141eb3294ae5a18872e6d62b5cba",
-  "step": 952
+  "saved_at": "2026-10-05T02:07:16Z",
+  "sha256": "a48a5b9502b31b009445854baf61187c25092cba10a3a810acd92237d34b4759",
+  "step": 1768
 }

repro_check/experiments/EXP_005_davidnet_s42/best_model/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4dddf8cbd5dc7a1bfddc3790ca23997d314141eb3294ae5a18872e6d62b5cba
+oid sha256:a48a5b9502b31b009445854baf61187c25092cba10a3a810acd92237d34b4759
 size 341484140

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/LATEST.json

@@ -1,10 +1,10 @@
 {
-  "step": 1500,
-  "dir": "checkpoints/step-0001500",
-  "updated_at": "2026-10-05T02:05:28Z",
+  "step": 2000,
+  "dir": "checkpoints/step-0002000",
+  "updated_at": "2026-10-05T02:08:47Z",
   "config_hash": "504b19d07f0d",
   "history": [
-    "checkpoints/step-0001000",
-    "checkpoints/step-0001500"
+    "checkpoints/step-0001500",
+    "checkpoints/step-0002000"
   ]
 }

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/{step-0001000 → step-0002000}/meta.json

@@ -1,13 +1,13 @@
 {
   "config_hash": "504b19d07f0d",
-  "created_at": "2026-10-05T02:02:11Z",
-  "epoch": 7,
+  "created_at": "2026-10-05T02:08:44Z",
+  "epoch": 14,
   "files": {
     "training_state.pt": {
-      "bytes": 1003383763,
-      "sha256": "c0ab9851466b5f405857196895e1ae31f550c3c5ae170f475ed448ed17315c91"
+      "bytes": 1003388499,
+      "sha256": "0f61db237060fc3b6af0de227d5c1132566a29a3f5fb7155701074a808778996"
     }
   },
   "reason": "interval",
-  "step": 1000
+  "step": 2000
 }

repro_check/experiments/EXP_005_davidnet_s42/checkpoints/{step-0001000 → step-0002000}/training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0ab9851466b5f405857196895e1ae31f550c3c5ae170f475ed448ed17315c91
-size 1003383763
+oid sha256:0f61db237060fc3b6af0de227d5c1132566a29a3f5fb7155701074a808778996
+size 1003388499

repro_check/experiments/EXP_005_davidnet_s42/logs/pipeline.jsonl

@@ -291,3 +291,11 @@
 {"t": "2026-10-05T02:05:23Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 11/30 (epoch=11, step=1496, lr=7.539e-05, train/v=0.01133, train/a=0.001863, train/quad=0.04027, train/loc=0.04381, train/sync=3.475, train/dis=0.01851, train/total=0.4046, val/v_loss=0.07528, val/a_loss=0.002414, val/quad_loss=0.2887, val/loc_loss=0.2614, val/sync_loss=1.021, val/dis_loss=0.01046, val/loss=0.4559, val/video_auc=0.9757, val/video_f1@0.5=0.9465, val/audio_auc=1, val/audio_f1@0.5=0.9928, val/clip_auc=0.9842, val/clip_f1@0.5=0.9704, val/mean_auc=0.9878, val/quad_macro_f1=0.9033, epoch_seconds=51.5, val_seconds=2, best=0.9906, improved=False)", "epoch": 11, "step": 1496, "lr": 7.538885275878594e-05, "train/v": 0.011332837240950218, "train/a": 0.0018626836741692816, "train/quad": 0.040273989742512215, "train/loc": 0.04380577014714671, "train/sync": 3.474968712119495, "train/dis": 0.018506204997024992, "train/total": 0.4045829001594992, "val/v_loss": 0.075275970933338, "val/a_loss": 0.0024137285663022746, "val/quad_loss": 0.28867936444779235, "val/loc_loss": 0.26137485976020497, "val/sync_loss": 1.021100680033366, "val/dis_loss": 0.010457693495684199, "val/loss": 0.45587265657054055, "val/video_auc": 0.9756952380952382, "val/video_f1@0.5": 0.946524064171123, "val/audio_auc": 1.0, "val/audio_f1@0.5": 0.9928400954653938, "val/clip_auc": 0.9842054574638845, "val/clip_f1@0.5": 0.9703872437357631, "val/mean_auc": 0.9878476190476191, "val/quad_macro_f1": 0.9033292277833762, "epoch_seconds": 51.5, "val_seconds": 2.0, "best": 0.9905708030568781, "improved": false}
 {"t": "2026-10-05T02:05:23Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9878476190476191"}
 {"t": "2026-10-05T02:05:28Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1500 (interval) (step=1500, epoch=11, mb=1003)", "step": 1500, "epoch": 11, "mb": 1003.4}
+{"t": "2026-10-05T02:05:41Z", "level": "INFO", "event": "checkpoint_uploaded", "msg": "[checkpoint_uploaded] step 1500 verified on HF (step=1500, seconds=13.1, with_best=False)", "step": 1500, "seconds": 13.1, "with_best": false}
+{"t": "2026-10-05T02:06:24Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 12/30 (epoch=12, step=1632, lr=7.053e-05, train/v=0.01171, train/a=0.004137, train/quad=0.04935, train/loc=0.05206, train/sync=3.615, train/dis=0.01428, train/total=0.4294, val/v_loss=0.2619, val/a_loss=0.002816, val/quad_loss=0.9413, val/loc_loss=0.8407, val/sync_loss=0.8751, val/dis_loss=0.01054, val/loss=1.244, val/video_auc=0.9823, val/video_f1@0.5=0.9465, val/audio_auc=0.9999, val/audio_f1@0.5=0.9952, val/clip_auc=0.9846, val/clip_f1@0.5=0.9681, val/mean_auc=0.9911, val/quad_macro_f1=0.9023, epoch_seconds=59.6, val_seconds=2, best=0.9911, improved=True)", "epoch": 12, "step": 1632, "lr": 7.053078557670201e-05, "train/v": 0.011714340218532016, "train/a": 0.004136669342833155, "train/quad": 0.049353862706886825, "train/loc": 0.0520597700301818, "train/sync": 3.6146044485709248, "train/dis": 0.014276332783457987, "train/total": 0.42944591426674056, "val/v_loss": 0.2618835914052195, "val/a_loss": 0.002816174135508239, "val/quad_loss": 0.9412569519546297, "val/loc_loss": 0.8407315272423956, "val/sync_loss": 0.8750752343071831, "val/dis_loss": 0.010540883781181442, "val/loss": 1.2442556768655777, "val/video_auc": 0.9822857142857143, "val/video_f1@0.5": 0.9465020576131687, "val/audio_auc": 0.9999060385868204, "val/audio_f1@0.5": 0.9951923076923077, "val/clip_auc": 0.9846388443017657, "val/clip_f1@0.5": 0.9681093394077449, "val/mean_auc": 0.9910958764362674, "val/quad_macro_f1": 0.902343642485564, "epoch_seconds": 59.6, "val_seconds": 2.0, "best": 0.9910958764362674, "improved": true}
+{"t": "2026-10-05T02:06:24Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9910958764362674"}
+{"t": "2026-10-05T02:07:17Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 13/30 (epoch=13, step=1768, lr=6.543e-05, train/v=0.009582, train/a=0.001351, train/quad=0.03557, train/loc=0.03865, train/sync=3.328, train/dis=0.01118, train/total=0.3819, val/v_loss=0.2608, val/a_loss=0.01417, val/quad_loss=0.9419, val/loc_loss=0.8466, val/sync_loss=0.8516, val/dis_loss=0.0084, val/loss=1.255, val/video_auc=0.983, val/video_f1@0.5=0.9326, val/audio_auc=0.9999, val/audio_f1@0.5=0.9788, val/clip_auc=0.983, val/clip_f1@0.5=0.9558, val/mean_auc=0.9914, val/quad_macro_f1=0.8773, epoch_seconds=51.5, val_seconds=2, best=0.9914, improved=True)", "epoch": 13, "step": 1768, "lr": 6.542957179213817e-05, "train/v": 0.009581871650452205, "train/a": 0.001351030038519826, "train/quad": 0.03556745735798305, "train/loc": 0.03865269258120166, "train/sync": 3.327797730179394, "train/dis": 0.011177329437273061, "train/total": 0.3819404901827083, "val/v_loss": 0.2608416945570045, "val/a_loss": 0.014173940441019618, "val/quad_loss": 0.9419311616155837, "val/loc_loss": 0.8465634667211108, "val/sync_loss": 0.8515707121955024, "val/dis_loss": 0.008399744673321644, "val/loss": 1.2552600138717227, "val/video_auc": 0.9829809523809524, "val/video_f1@0.5": 0.9325842696629213, "val/audio_auc": 0.9999060385868204, "val/audio_f1@0.5": 0.9788235294117648, "val/clip_auc": 0.9830337078651685, "val/clip_f1@0.5": 0.9558139534883721, "val/mean_auc": 0.9914434954838864, "val/quad_macro_f1": 0.8773059217450675, "epoch_seconds": 51.5, "val_seconds": 2.0, "best": 0.9914434954838864, "improved": true}
+{"t": "2026-10-05T02:07:17Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9914434954838864"}
+{"t": "2026-10-05T02:08:08Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_005 epoch 14/30 (epoch=14, step=1904, lr=6.015e-05, train/v=0.00888, train/a=0.001839, train/quad=0.03189, train/loc=0.03509, train/sync=3.23, train/dis=0.01854, train/total=0.3691, val/v_loss=0.07927, val/a_loss=0.002506, val/quad_loss=0.2602, val/loc_loss=0.2509, val/sync_loss=0.8204, val/dis_loss=0.008652, val/loss=0.4203, val/video_auc=0.9825, val/video_f1@0.5=0.9589, val/audio_auc=1, val/audio_f1@0.5=0.9976, val/clip_auc=0.9784, val/clip_f1@0.5=0.972, val/mean_auc=0.9912, val/quad_macro_f1=0.9224, epoch_seconds=51.5, val_seconds=2, best=0.9914, improved=False)", "epoch": 14, "step": 1904, "lr": 6.014713323015169e-05, "train/v": 0.008879632213152303, "train/a": 0.001838806870854542, "train/quad": 0.03189476718591108, "train/loc": 0.035091972605989535, "train/sync": 3.2303158360369064, "train/dis": 0.01853826010312118, "train/total": 0.3690972277785049, "val/v_loss": 0.07926819576985306, "val/a_loss": 0.0025056454760993396, "val/quad_loss": 0.2602427245842086, "val/loc_loss": 0.2509180088010099, "val/sync_loss": 0.8203589651319716, "val/dis_loss": 0.008652220480144024, "val/loss": 0.4202553356687228, "val/video_auc": 0.9824666666666666, "val/video_f1@0.5": 0.9589403973509933, "val/audio_auc": 0.9999843397644701, "val/audio_f1@0.5": 0.9976019184652278, "val/clip_auc": 0.978394863563403, "val/clip_f1@0.5": 0.9720044792833147, "val/mean_auc": 0.9912255032155683, "val/quad_macro_f1": 0.9223593378157078, "epoch_seconds": 51.5, "val_seconds": 2.0, "best": 0.9914434954838864, "improved": false}
+{"t": "2026-10-05T02:08:08Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9912255032155683"}
+{"t": "2026-10-05T02:08:47Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 2000 (interval) (step=2000, epoch=14, mb=1003)", "step": 2000, "epoch": 14, "mb": 1003.4}

repro_check/experiments/EXP_005_davidnet_s42/metrics/training_history.csv

@@ -10,3 +10,6 @@ epoch,step,lr,train/v,train/a,train/quad,train/loc,train/sync,train/dis,train/to
 9,1224,8.411389244402088e-05,0.01627865637483945,0.003769871600463362,0.06330716210119564,0.06783826308845378,3.697898103910334,0.027000088997952202,0.45811106511117783,0.06283980564007328,0.0022539284816376595,0.22392648779269722,0.21778364520933893,0.9212481710645888,0.01134108550225695,0.37920773981346023,0.9800000000000001,0.9549071618037135,0.9999530192934102,0.9975903614457832,0.9827608346709471,0.9707207207207207,0.9899765096467051,0.9044558458067159,51.6,2.0
 10,1360,7.994480298371163e-05,0.01728180047361735,0.006978785372827106,0.08084832882227869,0.08244854963285274,3.564415174372056,0.02736562627764857,0.46508711313500123,0.07464678968406385,0.02452731575168501,0.3340199569033252,0.329137915538417,0.8780298762851291,0.020119365511669055,0.5205679693155818,0.9756476190476191,0.9536423841059603,0.9996711350538712,0.983451536643026,0.98,0.9718785151856018,0.9876593770507451,0.9001765815853837,51.5,2.0
 11,1496,7.538885275878594e-05,0.011332837240950218,0.0018626836741692816,0.040273989742512215,0.04380577014714671,3.474968712119495,0.018506204997024992,0.4045829001594992,0.075275970933338,0.0024137285663022746,0.28867936444779235,0.26137485976020497,1.021100680033366,0.010457693495684199,0.45587265657054055,0.9756952380952382,0.946524064171123,1.0,0.9928400954653938,0.9842054574638845,0.9703872437357631,0.9878476190476191,0.9033292277833762,51.5,2.0
+12,1632,7.053078557670201e-05,0.011714340218532016,0.004136669342833155,0.049353862706886825,0.0520597700301818,3.6146044485709248,0.014276332783457987,0.42944591426674056,0.2618835914052195,0.002816174135508239,0.9412569519546297,0.8407315272423956,0.8750752343071831,0.010540883781181442,1.2442556768655777,0.9822857142857143,0.9465020576131687,0.9999060385868204,0.9951923076923077,0.9846388443017657,0.9681093394077449,0.9910958764362674,0.902343642485564,59.6,2.0
+13,1768,6.542957179213817e-05,0.009581871650452205,0.001351030038519826,0.03556745735798305,0.03865269258120166,3.327797730179394,0.011177329437273061,0.3819404901827083,0.2608416945570045,0.014173940441019618,0.9419311616155837,0.8465634667211108,0.8515707121955024,0.008399744673321644,1.2552600138717227,0.9829809523809524,0.9325842696629213,0.9999060385868204,0.9788235294117648,0.9830337078651685,0.9558139534883721,0.9914434954838864,0.8773059217450675,51.5,2.0
+14,1904,6.014713323015169e-05,0.008879632213152303,0.001838806870854542,0.03189476718591108,0.035091972605989535,3.2303158360369064,0.01853826010312118,0.3690972277785049,0.07926819576985306,0.0025056454760993396,0.2602427245842086,0.2509180088010099,0.8203589651319716,0.008652220480144024,0.4202553356687228,0.9824666666666666,0.9589403973509933,0.9999843397644701,0.9976019184652278,0.978394863563403,0.9720044792833147,0.9912255032155683,0.9223593378157078,51.5,2.0