EXP_004_late-fusion_s42: checkpoint step 1000

repro_check/experiments/EXP_004_late-fusion_s42/best_model/meta.json

@@ -1,10 +1,10 @@
 {
   "config_hash": "d50d826ea858",
-  "epoch": 3,
+  "epoch": 7,
   "exp_id": "EXP_004",
-  "metric": 0.9404405764159836,
+  "metric": 0.9593785866413715,
   "metric_name": "val/mean_auc",
-  "saved_at": "2026-10-05T01:46:03Z",
-  "sha256": "75a2e2e2cb3c468ac74e4f50dc7ac3f1b264c1f6d02cdaa02b665853d44d50ed",
-  "step": 408
+  "saved_at": "2026-10-05T01:46:20Z",
+  "sha256": "850a332676d527a1203b695023ff0a44c3a97f75f51baa7da0cf62e4e9e2b696",
+  "step": 952
 }

repro_check/experiments/EXP_004_late-fusion_s42/best_model/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75a2e2e2cb3c468ac74e4f50dc7ac3f1b264c1f6d02cdaa02b665853d44d50ed
+oid sha256:850a332676d527a1203b695023ff0a44c3a97f75f51baa7da0cf62e4e9e2b696
 size 4759640

repro_check/experiments/EXP_004_late-fusion_s42/checkpoints/LATEST.json

@@ -1,9 +1,10 @@
 {
-  "step": 500,
-  "dir": "checkpoints/step-0000500",
-  "updated_at": "2026-10-05T01:46:06Z",
+  "step": 1000,
+  "dir": "checkpoints/step-0001000",
+  "updated_at": "2026-10-05T01:46:22Z",
   "config_hash": "d50d826ea858",
   "history": [
-    "checkpoints/step-0000500"
+    "checkpoints/step-0000500",
+    "checkpoints/step-0001000"
   ]
 }

repro_check/experiments/EXP_004_late-fusion_s42/checkpoints/step-0001000/meta.json

@@ -0,0 +1,13 @@
+{
+  "config_hash": "d50d826ea858",
+  "created_at": "2026-10-05T01:46:21Z",
+  "epoch": 7,
+  "files": {
+    "training_state.pt": {
+      "bytes": 14311932,
+      "sha256": "2718b1d190d40e43f6401392d29a33565ae551a81e669602ca91e2c5c521c669"
+    }
+  },
+  "reason": "interval",
+  "step": 1000
+}

repro_check/experiments/EXP_004_late-fusion_s42/checkpoints/step-0001000/training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2718b1d190d40e43f6401392d29a33565ae551a81e669602ca91e2c5c521c669
+size 14311932

repro_check/experiments/EXP_004_late-fusion_s42/logs/pipeline.jsonl

@@ -148,3 +148,13 @@
 {"t": "2026-10-05T01:46:03Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_004 epoch 3/30 (epoch=3, step=408, lr=0.0009932, train/v=0.0975, train/a=0.01118, train/quad=0.3904, train/total=0.3039, val/v_loss=0.1167, val/a_loss=0.02016, val/quad_loss=0.4765, val/loss=0.3751, val/video_auc=0.8822, val/video_f1@0.5=0.8805, val/audio_auc=0.9987, val/audio_f1@0.5=0.9834, val/clip_auc=0.8859, val/clip_f1@0.5=0.9266, val/mean_auc=0.9404, val/quad_macro_f1=0.7145, epoch_seconds=4, val_seconds=0.4, best=0.9404, improved=True)", "epoch": 3, "step": 408, "lr": 0.0009932488451843476, "train/v": 0.09750300917007468, "train/a": 0.011176104175426778, "train/quad": 0.39039833876578245, "train/total": 0.30387828334727707, "val/v_loss": 0.11670523871564203, "val/a_loss": 0.02015629693606671, "val/quad_loss": 0.47653739071554607, "val/loss": 0.37513024566902053, "val/video_auc": 0.8821809523809524, "val/video_f1@0.5": 0.8804733727810651, "val/audio_auc": 0.9987002004510148, "val/audio_f1@0.5": 0.9833729216152018, "val/clip_auc": 0.8859390048154093, "val/clip_f1@0.5": 0.9266247379454927, "val/mean_auc": 0.9404405764159836, "val/quad_macro_f1": 0.7145430978694658, "epoch_seconds": 4.0, "val_seconds": 0.4, "best": 0.9404405764159836, "improved": true}
 {"t": "2026-10-05T01:46:03Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9404405764159836"}
 {"t": "2026-10-05T01:46:06Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 500 (interval) (step=500, epoch=3, mb=14.3)", "step": 500, "epoch": 3, "mb": 14.3}
+{"t": "2026-10-05T01:46:08Z", "level": "INFO", "event": "checkpoint_uploaded", "msg": "[checkpoint_uploaded] step 500 verified on HF (step=500, seconds=2.1, with_best=True)", "step": 500, "seconds": 2.1, "with_best": true}
+{"t": "2026-10-05T01:46:08Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_004 epoch 4/30 (epoch=4, step=544, lr=0.0009813, train/v=0.08271, train/a=0.008667, train/quad=0.3338, train/total=0.2583, val/v_loss=0.1602, val/a_loss=0.01752, val/quad_loss=0.6198, val/loss=0.4876, val/video_auc=0.8924, val/video_f1@0.5=0.7887, val/audio_auc=0.9979, val/audio_f1@0.5=0.9834, val/clip_auc=0.9006, val/clip_f1@0.5=0.8484, val/mean_auc=0.9451, val/quad_macro_f1=0.7484, epoch_seconds=4.2, val_seconds=0.4, best=0.9451, improved=True)", "epoch": 4, "step": 544, "lr": 0.0009813226601530789, "train/v": 0.08271229735520833, "train/a": 0.00866703112542917, "train/quad": 0.3337830196627799, "train/total": 0.2582708377500667, "val/v_loss": 0.1602078378200531, "val/a_loss": 0.01751828032795149, "val/quad_loss": 0.6197942660914527, "val/loss": 0.48762324783537125, "val/video_auc": 0.8923619047619047, "val/video_f1@0.5": 0.7887323943661971, "val/audio_auc": 0.9979015284389877, "val/audio_f1@0.5": 0.9833729216152018, "val/clip_auc": 0.9006260032102728, "val/clip_f1@0.5": 0.8484076433121019, "val/mean_auc": 0.9451317166004463, "val/quad_macro_f1": 0.7484101746112781, "epoch_seconds": 4.2, "val_seconds": 0.4, "best": 0.9451317166004463, "improved": true}
+{"t": "2026-10-05T01:46:08Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9451317166004463"}
+{"t": "2026-10-05T01:46:12Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_004 epoch 5/30 (epoch=5, step=680, lr=0.0009636, train/v=0.07245, train/a=0.005278, train/quad=0.2858, train/total=0.2207, val/v_loss=0.126, val/a_loss=0.03253, val/quad_loss=0.58, val/loss=0.4485, val/video_auc=0.9031, val/video_f1@0.5=0.8666, val/audio_auc=0.9971, val/audio_f1@0.5=0.9858, val/clip_auc=0.9001, val/clip_f1@0.5=0.9097, val/mean_auc=0.9501, val/quad_macro_f1=0.7881, epoch_seconds=4, val_seconds=0.4, best=0.9501, improved=True)", "epoch": 5, "step": 680, "lr": 0.0009636145632713268, "train/v": 0.07245332366951249, "train/a": 0.005277643379914106, "train/quad": 0.28584296212476845, "train/total": 0.2206524488351801, "val/v_loss": 0.12595788886149725, "val/a_loss": 0.03252786814771172, "val/quad_loss": 0.580036986205313, "val/loss": 0.4485042525662316, "val/video_auc": 0.9031333333333333, "val/video_f1@0.5": 0.866571018651363, "val/audio_auc": 0.9970715359559008, "val/audio_f1@0.5": 0.985781990521327, "val/clip_auc": 0.9001123595505618, "val/clip_f1@0.5": 0.9097387173396675, "val/mean_auc": 0.9501024346446171, "val/quad_macro_f1": 0.7880656483667968, "epoch_seconds": 4.0, "val_seconds": 0.4, "best": 0.9501024346446171, "improved": true}
+{"t": "2026-10-05T01:46:12Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9501024346446171"}
+{"t": "2026-10-05T01:46:16Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_004 epoch 6/30 (epoch=6, step=816, lr=0.0009403, train/v=0.06817, train/a=0.00458, train/quad=0.2676, train/total=0.2066, val/v_loss=0.1107, val/a_loss=0.02884, val/quad_loss=0.4919, val/loss=0.3855, val/video_auc=0.8983, val/video_f1@0.5=0.8571, val/audio_auc=0.9966, val/audio_f1@0.5=0.9858, val/clip_auc=0.902, val/clip_f1@0.5=0.9177, val/mean_auc=0.9474, val/quad_macro_f1=0.7786, epoch_seconds=4, val_seconds=0.4, best=0.9501, improved=False)", "epoch": 6, "step": 816, "lr": 0.000940339506847212, "train/v": 0.06816867273300886, "train/a": 0.00457952612417584, "train/quad": 0.2676085585399586, "train/total": 0.20655247853959308, "val/v_loss": 0.11068797152903345, "val/a_loss": 0.028837678859367263, "val/quad_loss": 0.4918620702293184, "val/loss": 0.3854566779401567, "val/video_auc": 0.8982571428571429, "val/video_f1@0.5": 0.8571428571428571, "val/audio_auc": 0.9965547481834127, "val/audio_f1@0.5": 0.985781990521327, "val/clip_auc": 0.9019582664526484, "val/clip_f1@0.5": 0.9177288528389339, "val/mean_auc": 0.9474059455202778, "val/quad_macro_f1": 0.7785694162397445, "epoch_seconds": 4.0, "val_seconds": 0.4, "best": 0.9501024346446171, "improved": false}
+{"t": "2026-10-05T01:46:16Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9474059455202778"}
+{"t": "2026-10-05T01:46:20Z", "level": "INFO", "event": "epoch_completed", "msg": "[epoch_completed] EXP_004 epoch 7/30 (epoch=7, step=952, lr=0.0009118, train/v=0.0606, train/a=0.006873, train/quad=0.2448, train/total=0.1899, val/v_loss=0.08224, val/a_loss=0.01904, val/quad_loss=0.3558, val/loss=0.2792, val/video_auc=0.9206, val/video_f1@0.5=0.9098, val/audio_auc=0.9981, val/audio_f1@0.5=0.9834, val/clip_auc=0.9197, val/clip_f1@0.5=0.9359, val/mean_auc=0.9594, val/quad_macro_f1=0.7847, epoch_seconds=4, val_seconds=0.4, best=0.9594, improved=True)", "epoch": 7, "step": 952, "lr": 0.0009117800185364963, "train/v": 0.060599811347749305, "train/a": 0.006872993936587377, "train/quad": 0.24476677590214155, "train/total": 0.18985619319274144, "val/v_loss": 0.08223676847087012, "val/a_loss": 0.019035375252466718, "val/quad_loss": 0.35576170020633274, "val/loss": 0.2791529864900642, "val/video_auc": 0.9206285714285714, "val/video_f1@0.5": 0.9097839898348158, "val/audio_auc": 0.9981286018541718, "val/audio_f1@0.5": 0.9833729216152018, "val/clip_auc": 0.9197431781701444, "val/clip_f1@0.5": 0.9359391965255157, "val/mean_auc": 0.9593785866413715, "val/quad_macro_f1": 0.7846527622876045, "epoch_seconds": 4.0, "val_seconds": 0.4, "best": 0.9593785866413715, "improved": true}
+{"t": "2026-10-05T01:46:20Z", "level": "INFO", "event": "validation_completed", "msg": "[validation_completed] val/mean_auc=0.9593785866413715"}
+{"t": "2026-10-05T01:46:21Z", "level": "INFO", "event": "checkpoint_saved", "msg": "[checkpoint_saved] step 1000 (interval) (step=1000, epoch=7, mb=14.3)", "step": 1000, "epoch": 7, "mb": 14.3}

repro_check/experiments/EXP_004_late-fusion_s42/metrics/training_history.csv

@@ -2,3 +2,7 @@ epoch,step,lr,train/v,train/a,train/quad,train/total,val/v_loss,val/a_loss,val/q
 1,136,0.0006715686274509804,0.12812178680563674,0.06490493329200785,0.7934251101139713,0.5897392737953102,0.11860161936945385,0.00994405700152533,0.5335113373067644,0.39530134449402493,0.8375619047619047,0.8648648648648649,0.9992326484590328,0.9785202863961814,0.8466613162118781,0.9291005291005291,0.9183972766104688,0.5609483802543579,4.2,0.4
 2,272,0.0009992483506344914,0.10951281695024055,0.014508814404989072,0.45885663742528243,0.35344994999468327,0.12368135237031513,0.026546104223598377,0.5114340136448542,0.40594445831245846,0.8588857142857144,0.8685258964143426,0.9969854046604861,0.9742388758782201,0.8759711075441412,0.9206349206349206,0.9279355594731002,0.7144518496402809,4.0,0.4
 3,408,0.0009932488451843476,0.09750300917007468,0.011176104175426778,0.39039833876578245,0.30387828334727707,0.11670523871564203,0.02015629693606671,0.47653739071554607,0.37513024566902053,0.8821809523809524,0.8804733727810651,0.9987002004510148,0.9833729216152018,0.8859390048154093,0.9266247379454927,0.9404405764159836,0.7145430978694658,4.0,0.4
+4,544,0.0009813226601530789,0.08271229735520833,0.00866703112542917,0.3337830196627799,0.2582708377500667,0.1602078378200531,0.01751828032795149,0.6197942660914527,0.48762324783537125,0.8923619047619047,0.7887323943661971,0.9979015284389877,0.9833729216152018,0.9006260032102728,0.8484076433121019,0.9451317166004463,0.7484101746112781,4.2,0.4
+5,680,0.0009636145632713268,0.07245332366951249,0.005277643379914106,0.28584296212476845,0.2206524488351801,0.12595788886149725,0.03252786814771172,0.580036986205313,0.4485042525662316,0.9031333333333333,0.866571018651363,0.9970715359559008,0.985781990521327,0.9001123595505618,0.9097387173396675,0.9501024346446171,0.7880656483667968,4.0,0.4
+6,816,0.000940339506847212,0.06816867273300886,0.00457952612417584,0.2676085585399586,0.20655247853959308,0.11068797152903345,0.028837678859367263,0.4918620702293184,0.3854566779401567,0.8982571428571429,0.8571428571428571,0.9965547481834127,0.985781990521327,0.9019582664526484,0.9177288528389339,0.9474059455202778,0.7785694162397445,4.0,0.4
+7,952,0.0009117800185364963,0.060599811347749305,0.006872993936587377,0.24476677590214155,0.18985619319274144,0.08223676847087012,0.019035375252466718,0.35576170020633274,0.2791529864900642,0.9206285714285714,0.9097839898348158,0.9981286018541718,0.9833729216152018,0.9197431781701444,0.9359391965255157,0.9593785866413715,0.7846527622876045,4.0,0.4