Verified D45/E45 source, measured progress and independent recovery checkpoints

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/BEST.json

@@ -1,10 +1,10 @@
 {
   "batch_next": 0,
   "bytes": 67597675,
-  "epoch_index": 15,
-  "global_step": 45,
-  "path": "checkpoints/history/step_0000045_epoch_015_20261105T023238Z.pt",
+  "epoch_index": 35,
+  "global_step": 105,
+  "path": "checkpoints/history/step_0000105_epoch_035_20261105T023238Z.pt",
   "remote_verified": false,
-  "sha256": "b7262236a83151e746380901af190e25ed64ce47f42aa09e83216d310e9a982d",
-  "timestamp_utc": "2026-11-05T02:44:42Z"
+  "sha256": "b9d4314f78b1d4534de9ce1956f19de931937390df93b86821005839fd3ce932",
+  "timestamp_utc": "2026-11-05T03:01:03Z"
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/CURRENT.json

@@ -1,10 +1,10 @@
 {
   "batch_next": 0,
   "bytes": 67597675,
-  "epoch_index": 27,
-  "global_step": 81,
-  "path": "checkpoints/history/step_0000081_epoch_027_20261105T023238Z.pt",
+  "epoch_index": 39,
+  "global_step": 117,
+  "path": "checkpoints/history/step_0000117_epoch_039_20261105T023238Z.pt",
   "remote_verified": false,
-  "sha256": "0fbbb10cf38344be11353a956869f32988435d145181c9491712729d9891a019",
-  "timestamp_utc": "2026-11-05T02:54:29Z"
+  "sha256": "0edcb691a6c3751d1c6d5655909f97eabb6d50d7e5251ce4cb5c6f69e10148ce",
+  "timestamp_utc": "2026-11-05T03:04:18Z"
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/PREVIOUS.json

@@ -1,10 +1,10 @@
 {
   "batch_next": 0,
   "bytes": 67597675,
-  "epoch_index": 26,
-  "global_step": 78,
-  "path": "checkpoints/history/step_0000078_epoch_026_20261105T023238Z.pt",
+  "epoch_index": 38,
+  "global_step": 114,
+  "path": "checkpoints/history/step_0000114_epoch_038_20261105T023238Z.pt",
   "remote_verified": false,
-  "sha256": "8f9b5f821d278ff714fc2c99b599de83068b7368f8c78144f66622bdc453c83a",
-  "timestamp_utc": "2026-11-05T02:53:40Z"
+  "sha256": "044f8b887a6c6e9b8348d361296e2c513fdd60f594b9d89a469c18b84079a07e",
+  "timestamp_utc": "2026-11-05T03:03:28Z"
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000105_epoch_035_20261105T023238Z.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9d4314f78b1d4534de9ce1956f19de931937390df93b86821005839fd3ce932
+size 67597675

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000114_epoch_038_20261105T023238Z.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:044f8b887a6c6e9b8348d361296e2c513fdd60f594b9d89a469c18b84079a07e
+size 67597675

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000117_epoch_039_20261105T023238Z.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0edcb691a6c3751d1c6d5655909f97eabb6d50d7e5251ce4cb5c6f69e10148ce
+size 67597675

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/finetuning/progress.json

@@ -1,9 +1,9 @@
 {
-  "best_epoch": 15,
-  "best_validation_accuracy": 0.9046296296296297,
-  "best_validation_macro_f1": 0.9015550896572803,
-  "completed_epochs": 27,
-  "global_step": 81,
-  "latest_validation_accuracy": 0.894074074074074,
-  "latest_validation_macro_f1": 0.8900212574493482
+  "best_epoch": 35,
+  "best_validation_accuracy": 0.9112962962962963,
+  "best_validation_macro_f1": 0.9080724563633671,
+  "completed_epochs": 39,
+  "global_step": 117,
+  "latest_validation_accuracy": 0.8925925925925926,
+  "latest_validation_macro_f1": 0.8883567923526862
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/epoch_metrics.csv

@@ -26,3 +26,15 @@ timestamp_utc,epoch,global_step,train_loss_mean,validation_accuracy,validation_m
 2026-11-05T02:52:52Z,25,75,0.036723836014668144,0.8983333333333333,0.8936804913470684,0.9046296296296297,15,49.3401894569397,checkpoints/history/step_0000075_epoch_025_20261105T023238Z.pt,5ec7adb0827cba0671a8ac604b5d791d0e718696e5d248f7e83dbaea4ce4eabb
 2026-11-05T02:53:40Z,26,78,0.019097846001386642,0.8946296296296297,0.8909698490644713,0.9046296296296297,15,48.29756045341492,checkpoints/history/step_0000078_epoch_026_20261105T023238Z.pt,8f9b5f821d278ff714fc2c99b599de83068b7368f8c78144f66622bdc453c83a
 2026-11-05T02:54:29Z,27,81,0.017922512255609035,0.894074074074074,0.8900212574493482,0.9046296296296297,15,48.7632040977478,checkpoints/history/step_0000081_epoch_027_20261105T023238Z.pt,0fbbb10cf38344be11353a956869f32988435d145181c9491712729d9891a019
+2026-11-05T02:55:18Z,28,84,0.016132629476487637,0.899074074074074,0.8951622639180801,0.9046296296296297,15,49.36452889442444,checkpoints/history/step_0000084_epoch_028_20261105T023238Z.pt,8305abb59ef2c1e54962f58de5165800fa707117f942b5d18abbf3c93c29e26c
+2026-11-05T02:56:08Z,29,87,0.015720718850692112,0.8959259259259259,0.8920434961263167,0.9046296296296297,15,49.33513331413269,checkpoints/history/step_0000087_epoch_029_20261105T023238Z.pt,b20ac5adebf639af02dd9734c5612e9435b925649bea1a2649e0a3125eefa024
+2026-11-05T02:56:56Z,30,90,0.02942262962460518,0.9011111111111111,0.8969226628693813,0.9046296296296297,15,48.81232976913452,checkpoints/history/step_0000090_epoch_030_20261105T023238Z.pt,67f6244f53cc84b0bf9c16dacddf576c2f97162ed0d2ab2bbf6f67767997569b
+2026-11-05T02:57:46Z,31,93,0.022038570915659268,0.905,0.9010489328944151,0.905,31,49.30660057067871,checkpoints/history/step_0000093_epoch_031_20261105T023238Z.pt,0abe8a3e74be3164b90b9904d7d1156b89f7b04a47398346edf7dfd9e940bf60
+2026-11-05T02:58:35Z,32,96,0.008526269967357317,0.9070370370370371,0.9035105644734378,0.9070370370370371,32,49.326499938964844,checkpoints/history/step_0000096_epoch_032_20261105T023238Z.pt,427ff4df6a16d86e1a0d92adf8f0b85ce39479c9d7da836a02b5d1170792daf8
+2026-11-05T02:59:25Z,33,99,0.008715117815881968,0.9042592592592592,0.9005146117019297,0.9070370370370371,32,49.63530921936035,checkpoints/history/step_0000099_epoch_033_20261105T023238Z.pt,cc70532ebb6a929e863264057049c0712e0c0271543b4bf4d4604b71e9812928
+2026-11-05T03:00:14Z,34,102,0.009365194321920475,0.9107407407407407,0.9072252438359187,0.9107407407407407,34,49.28120136260986,checkpoints/history/step_0000102_epoch_034_20261105T023238Z.pt,9711bb86b5cda628cb72e0398688d2a655dd97ddfc9da893d663b79b6e530dcc
+2026-11-05T03:01:03Z,35,105,0.0109775485470891,0.9112962962962963,0.9080724563633671,0.9112962962962963,35,48.8663010597229,checkpoints/history/step_0000105_epoch_035_20261105T023238Z.pt,b9d4314f78b1d4534de9ce1956f19de931937390df93b86821005839fd3ce932
+2026-11-05T03:01:52Z,36,108,0.007733953185379505,0.9083333333333333,0.9051260887878939,0.9112962962962963,35,48.82949471473694,checkpoints/history/step_0000108_epoch_036_20261105T023238Z.pt,df438a13cda980697dc4c1f5c141a4c581b3bdd883b3870420449541db60caa7
+2026-11-05T03:02:40Z,37,111,0.012900625355541706,0.9027777777777778,0.899717099889544,0.9112962962962963,35,48.23412346839905,checkpoints/history/step_0000111_epoch_037_20261105T023238Z.pt,e67ad43b1c8da741407fc16cb261155e7c5dea4fe762b1b723106658af4685fc
+2026-11-05T03:03:28Z,38,114,0.00527526702110966,0.8944444444444445,0.8904234418123396,0.9112962962962963,35,48.58493733406067,checkpoints/history/step_0000114_epoch_038_20261105T023238Z.pt,044f8b887a6c6e9b8348d361296e2c513fdd60f594b9d89a469c18b84079a07e
+2026-11-05T03:04:18Z,39,117,0.012061839457601309,0.8925925925925926,0.8883567923526862,0.9112962962962963,35,49.194804430007935,checkpoints/history/step_0000117_epoch_039_20261105T023238Z.pt,0edcb691a6c3751d1c6d5655909f97eabb6d50d7e5251ce4cb5c6f69e10148ce

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/events.jsonl

@@ -136,3 +136,66 @@
 {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317134857177734, "epoch": 27, "global_step": 81, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.9574737548828125, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.04176785424351692, "loss_real": 0.0054395608603954315, "loss_reg_real": 0.02814420498907566, "loss_reg_synthetic": 0.10715510696172714, "loss_synthetic": 0.06417709589004517, "loss_total": 0.025720637291669846, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.426917552947998, "timestamp_utc": "2026-11-05T02:53:54Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.68551793698329}
 {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 27, "global_step": 81, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000081_epoch_027_20261105T023238Z.pt", "remote_verified": false, "sha256": "0fbbb10cf38344be11353a956869f32988435d145181c9491712729d9891a019", "timestamp_utc": "2026-11-05T02:54:29Z"}
 {"arm": "spc_k0", "best_epoch": 15, "best_validation_accuracy": 0.9046296296296297, "checkpoint": "checkpoints/history/step_0000081_epoch_027_20261105T023238Z.pt", "checkpoint_sha256": "0fbbb10cf38344be11353a956869f32988435d145181c9491712729d9891a019", "duration_seconds": 48.7632040977478, "epoch": 27, "global_step": 81, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:54:29Z", "train_loss_mean": 0.017922512255609035, "validation_accuracy": 0.894074074074074, "validation_macro_f1": 0.8900212574493482}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317134857177734, "epoch": 28, "global_step": 82, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.2750052213668823, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.012819808907806873, "loss_real": 0.005020058248192072, "loss_reg_real": 0.030307834967970848, "loss_reg_synthetic": 0.06774300336837769, "loss_synthetic": 0.019947214052081108, "loss_total": 0.01285865530371666, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.777144908905029, "timestamp_utc": "2026-11-05T02:54:34Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.794246865192733}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317165374755859, "epoch": 28, "global_step": 83, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.6743941903114319, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.019400760531425476, "loss_real": 0.004599623382091522, "loss_reg_real": 0.02767566218972206, "loss_reg_synthetic": 0.08792620152235031, "loss_synthetic": 0.02747553400695324, "loss_total": 0.014900357462465763, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.622050523757935, "timestamp_utc": "2026-11-05T02:54:38Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.69333639735514}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317165374755859, "epoch": 28, "global_step": 84, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.3501663208007812, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.03530383110046387, "loss_real": 0.004903513938188553, "loss_reg_real": 0.02920493483543396, "loss_reg_synthetic": 0.11519787460565567, "loss_synthetic": 0.0436641164124012, "loss_total": 0.020638875663280487, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.532293796539307, "timestamp_utc": "2026-11-05T02:54:43Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.18132766973361}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 28, "global_step": 84, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000084_epoch_028_20261105T023238Z.pt", "remote_verified": false, "sha256": "8305abb59ef2c1e54962f58de5165800fa707117f942b5d18abbf3c93c29e26c", "timestamp_utc": "2026-11-05T02:55:18Z"}
+{"arm": "spc_k0", "best_epoch": 15, "best_validation_accuracy": 0.9046296296296297, "checkpoint": "checkpoints/history/step_0000084_epoch_028_20261105T023238Z.pt", "checkpoint_sha256": "8305abb59ef2c1e54962f58de5165800fa707117f942b5d18abbf3c93c29e26c", "duration_seconds": 49.36452889442444, "epoch": 28, "global_step": 84, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:55:18Z", "train_loss_mean": 0.016132629476487637, "validation_accuracy": 0.899074074074074, "validation_macro_f1": 0.8951622639180801}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317165374755859, "epoch": 29, "global_step": 85, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.725552499294281, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.012963769026100636, "loss_real": 0.004429812543094158, "loss_reg_real": 0.026423955336213112, "loss_reg_synthetic": 0.06569965183734894, "loss_synthetic": 0.015962090343236923, "loss_total": 0.011289558373391628, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.730393886566162, "timestamp_utc": "2026-11-05T02:55:23Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.05905746316538}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317165374755859, "epoch": 29, "global_step": 86, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.170062065124512, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.057934075593948364, "loss_real": 0.0043474845588207245, "loss_reg_real": 0.026513272896409035, "loss_reg_synthetic": 0.15622550249099731, "loss_synthetic": 0.0481223426759243, "loss_total": 0.023950401693582535, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.787984371185303, "timestamp_utc": "2026-11-05T02:55:28Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.733587680511288}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317230224609375, "epoch": 29, "global_step": 87, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.5770101547241211, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.013181080110371113, "loss_real": 0.004177977330982685, "loss_reg_real": 0.023734236136078835, "loss_reg_synthetic": 0.07431252300739288, "loss_synthetic": 0.019952457398176193, "loss_total": 0.011922196485102177, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.467257499694824, "timestamp_utc": "2026-11-05T02:55:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.489694741473517}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 29, "global_step": 87, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000087_epoch_029_20261105T023238Z.pt", "remote_verified": false, "sha256": "b20ac5adebf639af02dd9734c5612e9435b925649bea1a2649e0a3125eefa024", "timestamp_utc": "2026-11-05T02:56:08Z"}
+{"arm": "spc_k0", "best_epoch": 15, "best_validation_accuracy": 0.9046296296296297, "checkpoint": "checkpoints/history/step_0000087_epoch_029_20261105T023238Z.pt", "checkpoint_sha256": "b20ac5adebf639af02dd9734c5612e9435b925649bea1a2649e0a3125eefa024", "duration_seconds": 49.33513331413269, "epoch": 29, "global_step": 87, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:56:08Z", "train_loss_mean": 0.015720718850692112, "validation_accuracy": 0.8959259259259259, "validation_macro_f1": 0.8920434961263167}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317230224609375, "epoch": 30, "global_step": 88, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.1121973991394043, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.10793252289295197, "loss_real": 0.0037271264009177685, "loss_reg_real": 0.02471807971596718, "loss_reg_synthetic": 0.16820862889289856, "loss_synthetic": 0.13362352550029755, "loss_total": 0.04559797793626785, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.774379730224609, "timestamp_utc": "2026-11-05T02:56:12Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.809765295727384}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317230224609375, "epoch": 30, "global_step": 89, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.398404836654663, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.056033652275800705, "loss_real": 0.004369927104562521, "loss_reg_real": 0.024423714727163315, "loss_reg_synthetic": 0.10678251087665558, "loss_synthetic": 0.049234192818403244, "loss_total": 0.02293272875249386, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.596220254898071, "timestamp_utc": "2026-11-05T02:56:17Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.848970001729956}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317226409912109, "epoch": 30, "global_step": 90, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.571662425994873, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.03818152844905853, "loss_real": 0.003967241384088993, "loss_reg_real": 0.023576511070132256, "loss_reg_synthetic": 0.1272597312927246, "loss_synthetic": 0.04231766238808632, "loss_total": 0.019737182185053825, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.592768907546997, "timestamp_utc": "2026-11-05T02:56:22Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 20.902423338184832}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 30, "global_step": 90, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000090_epoch_030_20261105T023238Z.pt", "remote_verified": false, "sha256": "67f6244f53cc84b0bf9c16dacddf576c2f97162ed0d2ab2bbf6f67767997569b", "timestamp_utc": "2026-11-05T02:56:56Z"}
+{"arm": "spc_k0", "best_epoch": 15, "best_validation_accuracy": 0.9046296296296297, "checkpoint": "checkpoints/history/step_0000090_epoch_030_20261105T023238Z.pt", "checkpoint_sha256": "67f6244f53cc84b0bf9c16dacddf576c2f97162ed0d2ab2bbf6f67767997569b", "duration_seconds": 48.81232976913452, "epoch": 30, "global_step": 90, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:56:56Z", "train_loss_mean": 0.02942262962460518, "validation_accuracy": 0.9011111111111111, "validation_macro_f1": 0.8969226628693813}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317234039306641, "epoch": 31, "global_step": 91, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.058710098266602, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.05606071278452873, "loss_real": 0.004384322091937065, "loss_reg_real": 0.026750756427645683, "loss_reg_synthetic": 0.14188255369663239, "loss_synthetic": 0.04683670029044151, "loss_total": 0.02326541393995285, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6271374225616455, "timestamp_utc": "2026-11-05T02:57:01Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.662891397147543}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317237854003906, "epoch": 31, "global_step": 92, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.2615907192230225, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.019631363451480865, "loss_real": 0.004302102141082287, "loss_reg_real": 0.025406112894415855, "loss_reg_synthetic": 0.08588998764753342, "loss_synthetic": 0.02063550055027008, "loss_total": 0.013126382604241371, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.750658988952637, "timestamp_utc": "2026-11-05T02:57:06Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.9436304095192}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317241668701172, "epoch": 31, "global_step": 93, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.148382544517517, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.07080309838056564, "loss_real": 0.0033148175571113825, "loss_reg_real": 0.0224150400608778, "loss_reg_synthetic": 0.12381251901388168, "loss_synthetic": 0.07977968454360962, "loss_total": 0.029723916202783585, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.413527965545654, "timestamp_utc": "2026-11-05T02:57:10Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.751306607645184}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 31, "global_step": 93, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000093_epoch_031_20261105T023238Z.pt", "remote_verified": false, "sha256": "0abe8a3e74be3164b90b9904d7d1156b89f7b04a47398346edf7dfd9e940bf60", "timestamp_utc": "2026-11-05T02:57:46Z"}
+{"arm": "spc_k0", "best_epoch": 31, "best_validation_accuracy": 0.905, "checkpoint": "checkpoints/history/step_0000093_epoch_031_20261105T023238Z.pt", "checkpoint_sha256": "0abe8a3e74be3164b90b9904d7d1156b89f7b04a47398346edf7dfd9e940bf60", "duration_seconds": 49.30660057067871, "epoch": 31, "global_step": 93, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:57:46Z", "train_loss_mean": 0.022038570915659268, "validation_accuracy": 0.905, "validation_macro_f1": 0.9010489328944151}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317279815673828, "epoch": 32, "global_step": 94, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.110559105873108, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.008469532243907452, "loss_real": 0.0033917527180165052, "loss_reg_real": 0.02232917957007885, "loss_reg_synthetic": 0.0478820875287056, "loss_synthetic": 0.012742727063596249, "loss_total": 0.008638059720396996, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6736369132995605, "timestamp_utc": "2026-11-05T02:57:50Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.387664547871935}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317279815673828, "epoch": 32, "global_step": 95, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.0676622986793518, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0042305937968194485, "loss_real": 0.0036468247417360544, "loss_reg_real": 0.021090226247906685, "loss_reg_synthetic": 0.03534434363245964, "loss_synthetic": 0.007645174860954285, "loss_total": 0.007149415090680122, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.762410879135132, "timestamp_utc": "2026-11-05T02:57:55Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.877143373073512}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317279815673828, "epoch": 32, "global_step": 96, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.130402684211731, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.014490208588540554, "loss_real": 0.003454978810623288, "loss_reg_real": 0.02332036755979061, "loss_reg_synthetic": 0.06503916531801224, "loss_synthetic": 0.013268684968352318, "loss_total": 0.009791335090994835, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.494978427886963, "timestamp_utc": "2026-11-05T02:58:00Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.357165899710107}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 32, "global_step": 96, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000096_epoch_032_20261105T023238Z.pt", "remote_verified": false, "sha256": "427ff4df6a16d86e1a0d92adf8f0b85ce39479c9d7da836a02b5d1170792daf8", "timestamp_utc": "2026-11-05T02:58:35Z"}
+{"arm": "spc_k0", "best_epoch": 32, "best_validation_accuracy": 0.9070370370370371, "checkpoint": "checkpoints/history/step_0000096_epoch_032_20261105T023238Z.pt", "checkpoint_sha256": "427ff4df6a16d86e1a0d92adf8f0b85ce39479c9d7da836a02b5d1170792daf8", "duration_seconds": 49.326499938964844, "epoch": 32, "global_step": 96, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:58:35Z", "train_loss_mean": 0.008526269967357317, "validation_accuracy": 0.9070370370370371, "validation_macro_f1": 0.9035105644734378}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317279815673828, "epoch": 33, "global_step": 97, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.08405602723360062, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.004277386702597141, "loss_real": 0.0033757449127733707, "loss_reg_real": 0.02105969563126564, "loss_reg_synthetic": 0.036890409886837006, "loss_synthetic": 0.006960364989936352, "loss_total": 0.006775752641260624, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.751831769943237, "timestamp_utc": "2026-11-05T02:58:40Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.936980557611157}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317279815673828, "epoch": 33, "global_step": 98, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.12757274508476257, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0054017771035432816, "loss_real": 0.003101211041212082, "loss_reg_real": 0.021595196798443794, "loss_reg_synthetic": 0.04132545739412308, "loss_synthetic": 0.00860578753054142, "loss_total": 0.007052863482385874, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.692321062088013, "timestamp_utc": "2026-11-05T02:58:45Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.2786107997141}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317295074462891, "epoch": 33, "global_step": 99, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.155389428138733, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.018710946664214134, "loss_real": 0.003391660051420331, "loss_reg_real": 0.020696062594652176, "loss_reg_synthetic": 0.07362044602632523, "loss_synthetic": 0.023768654093146324, "loss_total": 0.012316737323999405, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.563839673995972, "timestamp_utc": "2026-11-05T02:58:49Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.03491946638543}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 33, "global_step": 99, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000099_epoch_033_20261105T023238Z.pt", "remote_verified": false, "sha256": "cc70532ebb6a929e863264057049c0712e0c0271543b4bf4d4604b71e9812928", "timestamp_utc": "2026-11-05T02:59:25Z"}
+{"arm": "spc_k0", "best_epoch": 32, "best_validation_accuracy": 0.9070370370370371, "checkpoint": "checkpoints/history/step_0000099_epoch_033_20261105T023238Z.pt", "checkpoint_sha256": "cc70532ebb6a929e863264057049c0712e0c0271543b4bf4d4604b71e9812928", "duration_seconds": 49.63530921936035, "epoch": 33, "global_step": 99, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:59:25Z", "train_loss_mean": 0.008715117815881968, "validation_accuracy": 0.9042592592592592, "validation_macro_f1": 0.9005146117019297}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317295074462891, "epoch": 34, "global_step": 100, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.33037763833999634, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0066314274445176125, "loss_real": 0.0029871356673538685, "loss_reg_real": 0.020121244713664055, "loss_reg_synthetic": 0.04831748455762863, "loss_synthetic": 0.009983005002140999, "loss_total": 0.00741807883605361, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.725587368011475, "timestamp_utc": "2026-11-05T02:59:29Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.086579938498176}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317295074462891, "epoch": 34, "global_step": 101, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.65028315782547, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.01076524704694748, "loss_real": 0.00336146866902709, "loss_reg_real": 0.022330215200781822, "loss_reg_synthetic": 0.06363549083471298, "loss_synthetic": 0.013590243645012379, "loss_total": 0.009321960620582104, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.736891269683838, "timestamp_utc": "2026-11-05T02:59:34Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.021941757287436}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317295074462891, "epoch": 34, "global_step": 102, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.9175833463668823, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.01578894816339016, "loss_real": 0.0033056936226785183, "loss_reg_real": 0.021558966487646103, "loss_reg_synthetic": 0.07401445508003235, "loss_synthetic": 0.020641542971134186, "loss_total": 0.01135554350912571, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.422568321228027, "timestamp_utc": "2026-11-05T02:59:39Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.706843857947096}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 34, "global_step": 102, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000102_epoch_034_20261105T023238Z.pt", "remote_verified": false, "sha256": "9711bb86b5cda628cb72e0398688d2a655dd97ddfc9da893d663b79b6e530dcc", "timestamp_utc": "2026-11-05T03:00:14Z"}
+{"arm": "spc_k0", "best_epoch": 34, "best_validation_accuracy": 0.9107407407407407, "checkpoint": "checkpoints/history/step_0000102_epoch_034_20261105T023238Z.pt", "checkpoint_sha256": "9711bb86b5cda628cb72e0398688d2a655dd97ddfc9da893d663b79b6e530dcc", "duration_seconds": 49.28120136260986, "epoch": 34, "global_step": 102, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:00:14Z", "train_loss_mean": 0.009365194321920475, "validation_accuracy": 0.9107407407407407, "validation_macro_f1": 0.9072252438359187}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317291259765625, "epoch": 35, "global_step": 103, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.08429992198944092, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.004013978876173496, "loss_real": 0.0030498672276735306, "loss_reg_real": 0.020487092435359955, "loss_reg_synthetic": 0.03710741549730301, "loss_synthetic": 0.007094493135809898, "loss_total": 0.006431795656681061, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.771285772323608, "timestamp_utc": "2026-11-05T03:00:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.827150187163117}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317291259765625, "epoch": 35, "global_step": 104, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.8819658756256104, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.04953889548778534, "loss_real": 0.003329092636704445, "loss_reg_real": 0.021449590101838112, "loss_reg_synthetic": 0.11243458092212677, "loss_synthetic": 0.04320366308093071, "loss_total": 0.020030388608574867, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.551757574081421, "timestamp_utc": "2026-11-05T03:00:23Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.121005549341316}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317291259765625, "epoch": 35, "global_step": 105, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.17257003486156464, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.005397097673267126, "loss_real": 0.002896874211728573, "loss_reg_real": 0.019380679354071617, "loss_reg_synthetic": 0.04339449480175972, "loss_synthetic": 0.006953801494091749, "loss_total": 0.006470461376011372, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.577567100524902, "timestamp_utc": "2026-11-05T03:00:28Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 20.971838946717316}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 35, "global_step": 105, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000105_epoch_035_20261105T023238Z.pt", "remote_verified": false, "sha256": "b9d4314f78b1d4534de9ce1956f19de931937390df93b86821005839fd3ce932", "timestamp_utc": "2026-11-05T03:01:03Z"}
+{"arm": "spc_k0", "best_epoch": 35, "best_validation_accuracy": 0.9112962962962963, "checkpoint": "checkpoints/history/step_0000105_epoch_035_20261105T023238Z.pt", "checkpoint_sha256": "b9d4314f78b1d4534de9ce1956f19de931937390df93b86821005839fd3ce932", "duration_seconds": 48.8663010597229, "epoch": 35, "global_step": 105, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:01:03Z", "train_loss_mean": 0.0109775485470891, "validation_accuracy": 0.9112962962962963, "validation_macro_f1": 0.9080724563633671}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317295074462891, "epoch": 36, "global_step": 106, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.3869265615940094, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.004777777474373579, "loss_real": 0.0032099143136292696, "loss_reg_real": 0.020404072478413582, "loss_reg_synthetic": 0.03957637771964073, "loss_synthetic": 0.008326738141477108, "loss_total": 0.006960729602724314, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.604785442352295, "timestamp_utc": "2026-11-05T03:01:07Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.797169184632597}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317302703857422, "epoch": 36, "global_step": 107, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.697833240032196, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.012559023685753345, "loss_real": 0.0028733815997838974, "loss_reg_real": 0.01990017108619213, "loss_reg_synthetic": 0.06274174898862839, "loss_synthetic": 0.014910601079463959, "loss_total": 0.009162246249616146, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.682635068893433, "timestamp_utc": "2026-11-05T03:01:12Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.335036388015617}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317314147949219, "epoch": 36, "global_step": 108, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.24839337170124054, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0062500424683094025, "loss_real": 0.0029025920666754246, "loss_reg_real": 0.019115250557661057, "loss_reg_synthetic": 0.04537143185734749, "loss_synthetic": 0.009396244771778584, "loss_total": 0.007078883703798056, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.505893707275391, "timestamp_utc": "2026-11-05T03:01:17Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.305429341352344}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 36, "global_step": 108, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000108_epoch_036_20261105T023238Z.pt", "remote_verified": false, "sha256": "df438a13cda980697dc4c1f5c141a4c581b3bdd883b3870420449541db60caa7", "timestamp_utc": "2026-11-05T03:01:52Z"}
+{"arm": "spc_k0", "best_epoch": 35, "best_validation_accuracy": 0.9112962962962963, "checkpoint": "checkpoints/history/step_0000108_epoch_036_20261105T023238Z.pt", "checkpoint_sha256": "df438a13cda980697dc4c1f5c141a4c581b3bdd883b3870420449541db60caa7", "duration_seconds": 48.82949471473694, "epoch": 36, "global_step": 108, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:01:52Z", "train_loss_mean": 0.007733953185379505, "validation_accuracy": 0.9083333333333333, "validation_macro_f1": 0.9051260887878939}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317314147949219, "epoch": 37, "global_step": 109, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.895182132720947, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.037825729697942734, "loss_real": 0.002673803362995386, "loss_reg_real": 0.018608858808875084, "loss_reg_synthetic": 0.0891939103603363, "loss_synthetic": 0.05616578087210655, "loss_total": 0.020217763260006905, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.530768156051636, "timestamp_utc": "2026-11-05T03:01:56Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.251280045974884}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317314147949219, "epoch": 37, "global_step": 110, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.671847939491272, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.022040992975234985, "loss_real": 0.002435058355331421, "loss_reg_real": 0.016377722844481468, "loss_reg_synthetic": 0.05677291750907898, "loss_synthetic": 0.03549623116850853, "loss_total": 0.013528971932828426, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.784580707550049, "timestamp_utc": "2026-11-05T03:02:01Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.752605468231838}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317317962646484, "epoch": 37, "global_step": 111, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.17068901658058167, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.003183918772265315, "loss_real": 0.002207765821367502, "loss_reg_real": 0.01492322701960802, "loss_reg_synthetic": 0.0256162341684103, "loss_synthetic": 0.006598646752536297, "loss_total": 0.004955140873789787, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.384855270385742, "timestamp_utc": "2026-11-05T03:02:05Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.89353902929497}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 37, "global_step": 111, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000111_epoch_037_20261105T023238Z.pt", "remote_verified": false, "sha256": "e67ad43b1c8da741407fc16cb261155e7c5dea4fe762b1b723106658af4685fc", "timestamp_utc": "2026-11-05T03:02:40Z"}
+{"arm": "spc_k0", "best_epoch": 35, "best_validation_accuracy": 0.9112962962962963, "checkpoint": "checkpoints/history/step_0000111_epoch_037_20261105T023238Z.pt", "checkpoint_sha256": "e67ad43b1c8da741407fc16cb261155e7c5dea4fe762b1b723106658af4685fc", "duration_seconds": 48.23412346839905, "epoch": 37, "global_step": 111, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:02:40Z", "train_loss_mean": 0.012900625355541706, "validation_accuracy": 0.9027777777777778, "validation_macro_f1": 0.899717099889544}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317317962646484, "epoch": 38, "global_step": 112, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.4342043995857239, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.006294739432632923, "loss_real": 0.0020860431250184774, "loss_reg_real": 0.015052169561386108, "loss_reg_synthetic": 0.042329091578722, "loss_synthetic": 0.006718870718032122, "loss_total": 0.005507960449904203, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.769389867782593, "timestamp_utc": "2026-11-05T03:02:45Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.837814384738138}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317352294921875, "epoch": 38, "global_step": 113, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.488032341003418, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0073803323321044445, "loss_real": 0.0021975692361593246, "loss_reg_real": 0.014317098073661327, "loss_reg_synthetic": 0.041808780282735825, "loss_synthetic": 0.007395792752504349, "loss_total": 0.00582362012937665, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.571729898452759, "timestamp_utc": "2026-11-05T03:02:49Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.998154493623932}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317352294921875, "epoch": 38, "global_step": 114, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.46675992012023926, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0035414707381278276, "loss_real": 0.0019903164356946945, "loss_reg_real": 0.012403125874698162, "loss_reg_synthetic": 0.026121987029910088, "loss_synthetic": 0.005655961111187935, "loss_total": 0.004494220484048128, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.508204460144043, "timestamp_utc": "2026-11-05T03:02:54Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.29450890009826}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 38, "global_step": 114, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000114_epoch_038_20261105T023238Z.pt", "remote_verified": false, "sha256": "044f8b887a6c6e9b8348d361296e2c513fdd60f594b9d89a469c18b84079a07e", "timestamp_utc": "2026-11-05T03:03:28Z"}
+{"arm": "spc_k0", "best_epoch": 35, "best_validation_accuracy": 0.9112962962962963, "checkpoint": "checkpoints/history/step_0000114_epoch_038_20261105T023238Z.pt", "checkpoint_sha256": "044f8b887a6c6e9b8348d361296e2c513fdd60f594b9d89a469c18b84079a07e", "duration_seconds": 48.58493733406067, "epoch": 38, "global_step": 114, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:03:28Z", "train_loss_mean": 0.00527526702110966, "validation_accuracy": 0.8944444444444445, "validation_macro_f1": 0.8904234418123396}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317356109619141, "epoch": 39, "global_step": 115, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.3818507194519043, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.028782391920685768, "loss_real": 0.0019914451986551285, "loss_reg_real": 0.01404609065502882, "loss_reg_synthetic": 0.07711786776781082, "loss_synthetic": 0.02509084716439247, "loss_total": 0.011992054060101509, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.561222314834595, "timestamp_utc": "2026-11-05T03:03:33Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.062653202344887}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317356109619141, "epoch": 39, "global_step": 116, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.368604838848114, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.004938909318298101, "loss_real": 0.0018637379398569465, "loss_reg_real": 0.01286547165364027, "loss_reg_synthetic": 0.035925060510635376, "loss_synthetic": 0.006048420909792185, "loss_total": 0.00480043375864625, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.768991231918335, "timestamp_utc": "2026-11-05T03:03:38Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.840057734497403}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317352294921875, "epoch": 39, "global_step": 117, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.5854291915893555, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.050501544028520584, "loss_real": 0.0020835930481553078, "loss_reg_real": 0.013543962500989437, "loss_reg_synthetic": 0.09488000720739365, "loss_synthetic": 0.049099624156951904, "loss_total": 0.019393030554056168, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.431109666824341, "timestamp_utc": "2026-11-05T03:03:42Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.665002046496554}
+{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 39, "global_step": 117, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000117_epoch_039_20261105T023238Z.pt", "remote_verified": false, "sha256": "0edcb691a6c3751d1c6d5655909f97eabb6d50d7e5251ce4cb5c6f69e10148ce", "timestamp_utc": "2026-11-05T03:04:18Z"}
+{"arm": "spc_k0", "best_epoch": 35, "best_validation_accuracy": 0.9112962962962963, "checkpoint": "checkpoints/history/step_0000117_epoch_039_20261105T023238Z.pt", "checkpoint_sha256": "0edcb691a6c3751d1c6d5655909f97eabb6d50d7e5251ce4cb5c6f69e10148ce", "duration_seconds": 49.194804430007935, "epoch": 39, "global_step": 117, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T03:04:18Z", "train_loss_mean": 0.012061839457601309, "validation_accuracy": 0.8925925925925926, "validation_macro_f1": 0.8883567923526862}
+{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.317352294921875, "epoch": 40, "global_step": 118, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.11168286949396133, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.003772095311433077, "loss_real": 0.0018372742924839258, "loss_reg_real": 0.012820491567254066, "loss_reg_synthetic": 0.031674545258283615, "loss_synthetic": 0.004412771202623844, "loss_total": 0.004243348725140095, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.705322027206421, "timestamp_utc": "2026-11-05T03:04:22Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.203239068420235}
+{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.317352294921875, "epoch": 40, "global_step": 119, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.40780383348464966, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.004888775292783976, "loss_real": 0.001685254741460085, "loss_reg_real": 0.011777923442423344, "loss_reg_synthetic": 0.03290068730711937, "loss_synthetic": 0.006640085484832525, "loss_total": 0.004631279967725277, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.594045162200928, "timestamp_utc": "2026-11-05T03:04:27Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.862155351271603}
+{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.317352294921875, "epoch": 40, "global_step": 120, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 0.1261381357908249, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.0018322281539440155, "loss_real": 0.0018683280795812607, "loss_reg_real": 0.012461252510547638, "loss_reg_synthetic": 0.01953316293656826, "loss_synthetic": 0.0036391704343259335, "loss_total": 0.003668498480692506, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.653713226318359, "timestamp_utc": "2026-11-05T03:04:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 20.628688389539512}

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/train_steps.csv

@@ -80,3 +80,42 @@ timestamp_utc,train_run_id,recovery_segment_id,epoch,batch,global_step,loss_tota
 2026-11-05T02:53:45Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,27,1,79,0.01615871489048004,0.005069835111498833,0.02493012137711048,0.027166742831468582,0.03232859447598457,0.10465192794799805,0.0001,4.707437992095947,27.191011377084347,4.21419620513916,13.199567317962646,6.31707763671875,1.3423982858657837
 2026-11-05T02:53:50Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,27,2,80,0.01188818458467722,0.005878056865185499,0.013204324059188366,0.008195820264518261,0.03419795632362366,0.05908814072608948,0.0001,4.76188325881958,26.88012138116335,4.21370792388916,13.199567317962646,6.3170928955078125,0.36173203587532043
 2026-11-05T02:53:54Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,27,3,81,0.025720637291669846,0.0054395608603954315,0.06417709589004517,0.04176785424351692,0.02814420498907566,0.10715510696172714,0.0001,4.426917552947998,21.68551793698329,4.20784854888916,13.199567317962646,6.317134857177734,0.9574737548828125
+2026-11-05T02:54:34Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,28,1,82,0.01285865530371666,0.005020058248192072,0.019947214052081108,0.012819808907806873,0.030307834967970848,0.06774300336837769,0.0001,4.777144908905029,26.794246865192733,4.21370792388916,13.199567317962646,6.317134857177734,1.2750052213668823
+2026-11-05T02:54:38Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,28,2,83,0.014900357462465763,0.004599623382091522,0.02747553400695324,0.019400760531425476,0.02767566218972206,0.08792620152235031,0.0001,4.622050523757935,27.69333639735514,4.21419620513916,13.199567317962646,6.317165374755859,0.6743941903114319
+2026-11-05T02:54:43Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,28,3,84,0.020638875663280487,0.004903513938188553,0.0436641164124012,0.03530383110046387,0.02920493483543396,0.11519787460565567,0.0001,4.532293796539307,21.18132766973361,4.20736026763916,13.199567317962646,6.317165374755859,2.3501663208007812
+2026-11-05T02:55:23Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,29,1,85,0.011289558373391628,0.004429812543094158,0.015962090343236923,0.012963769026100636,0.026423955336213112,0.06569965183734894,0.0001,4.730393886566162,27.05905746316538,4.21419620513916,13.199567317962646,6.317165374755859,0.725552499294281
+2026-11-05T02:55:28Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,29,2,86,0.023950401693582535,0.0043474845588207245,0.0481223426759243,0.057934075593948364,0.026513272896409035,0.15622550249099731,0.0001,4.787984371185303,26.733587680511288,4.21370792388916,13.199567317962646,6.317165374755859,4.170062065124512
+2026-11-05T02:55:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,29,3,87,0.011922196485102177,0.004177977330982685,0.019952457398176193,0.013181080110371113,0.023734236136078835,0.07431252300739288,0.0001,4.467257499694824,21.489694741473517,4.20821475982666,13.199567317962646,6.317230224609375,0.5770101547241211
+2026-11-05T02:56:12Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,30,1,88,0.04559797793626785,0.0037271264009177685,0.13362352550029755,0.10793252289295197,0.02471807971596718,0.16820862889289856,0.0001,4.774379730224609,26.809765295727384,4.21370792388916,13.199567317962646,6.317230224609375,1.1121973991394043
+2026-11-05T02:56:17Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,30,2,89,0.02293272875249386,0.004369927104562521,0.049234192818403244,0.056033652275800705,0.024423714727163315,0.10678251087665558,0.0001,4.596220254898071,27.848970001729956,4.21419620513916,13.199567317962646,6.317230224609375,1.398404836654663
+2026-11-05T02:56:22Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,30,3,90,0.019737182185053825,0.003967241384088993,0.04231766238808632,0.03818152844905853,0.023576511070132256,0.1272597312927246,0.0001,4.592768907546997,20.902423338184832,4.20736026763916,13.199567317962646,6.317226409912109,2.571662425994873
+2026-11-05T02:57:01Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,31,1,91,0.02326541393995285,0.004384322091937065,0.04683670029044151,0.05606071278452873,0.026750756427645683,0.14188255369663239,0.0001,4.6271374225616455,27.662891397147543,4.21419620513916,13.199567317962646,6.317234039306641,5.058710098266602
+2026-11-05T02:57:06Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,31,2,92,0.013126382604241371,0.004302102141082287,0.02063550055027008,0.019631363451480865,0.025406112894415855,0.08588998764753342,0.0001,4.750658988952637,26.9436304095192,4.21370792388916,13.199567317962646,6.317237854003906,1.2615907192230225
+2026-11-05T02:57:10Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,31,3,93,0.029723916202783585,0.0033148175571113825,0.07977968454360962,0.07080309838056564,0.0224150400608778,0.12381251901388168,0.0001,4.413527965545654,21.751306607645184,4.20784854888916,13.199567317962646,6.317241668701172,1.148382544517517
+2026-11-05T02:57:50Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,32,1,94,0.008638059720396996,0.0033917527180165052,0.012742727063596249,0.008469532243907452,0.02232917957007885,0.0478820875287056,0.0001,4.6736369132995605,27.387664547871935,4.21370792388916,13.199567317962646,6.317279815673828,1.110559105873108
+2026-11-05T02:57:55Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,32,2,95,0.007149415090680122,0.0036468247417360544,0.007645174860954285,0.0042305937968194485,0.021090226247906685,0.03534434363245964,0.0001,4.762410879135132,26.877143373073512,4.21419620513916,13.199567317962646,6.317279815673828,0.0676622986793518
+2026-11-05T02:58:00Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,32,3,96,0.009791335090994835,0.003454978810623288,0.013268684968352318,0.014490208588540554,0.02332036755979061,0.06503916531801224,0.0001,4.494978427886963,21.357165899710107,4.20736026763916,13.199567317962646,6.317279815673828,1.130402684211731
+2026-11-05T02:58:40Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,33,1,97,0.006775752641260624,0.0033757449127733707,0.006960364989936352,0.004277386702597141,0.02105969563126564,0.036890409886837006,0.0001,4.751831769943237,26.936980557611157,4.21419620513916,13.199567317962646,6.317279815673828,0.08405602723360062
+2026-11-05T02:58:45Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,33,2,98,0.007052863482385874,0.003101211041212082,0.00860578753054142,0.0054017771035432816,0.021595196798443794,0.04132545739412308,0.0001,4.692321062088013,27.2786107997141,4.21370792388916,13.199567317962646,6.317279815673828,0.12757274508476257
+2026-11-05T02:58:49Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,33,3,99,0.012316737323999405,0.003391660051420331,0.023768654093146324,0.018710946664214134,0.020696062594652176,0.07362044602632523,0.0001,4.563839673995972,21.03491946638543,4.20821475982666,13.199567317962646,6.317295074462891,1.155389428138733
+2026-11-05T02:59:29Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,34,1,100,0.00741807883605361,0.0029871356673538685,0.009983005002140999,0.0066314274445176125,0.020121244713664055,0.04831748455762863,0.0001,4.725587368011475,27.086579938498176,4.21370792388916,13.199567317962646,6.317295074462891,0.33037763833999634
+2026-11-05T02:59:34Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,34,2,101,0.009321960620582104,0.00336146866902709,0.013590243645012379,0.01076524704694748,0.022330215200781822,0.06363549083471298,0.0001,4.736891269683838,27.021941757287436,4.21419620513916,13.199567317962646,6.317295074462891,0.65028315782547
+2026-11-05T02:59:39Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,34,3,102,0.01135554350912571,0.0033056936226785183,0.020641542971134186,0.01578894816339016,0.021558966487646103,0.07401445508003235,0.0001,4.422568321228027,21.706843857947096,4.20736026763916,13.199567317962646,6.317295074462891,0.9175833463668823
+2026-11-05T03:00:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,35,1,103,0.006431795656681061,0.0030498672276735306,0.007094493135809898,0.004013978876173496,0.020487092435359955,0.03710741549730301,0.0001,4.771285772323608,26.827150187163117,4.21419620513916,13.199567317962646,6.317291259765625,0.08429992198944092
+2026-11-05T03:00:23Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,35,2,104,0.020030388608574867,0.003329092636704445,0.04320366308093071,0.04953889548778534,0.021449590101838112,0.11243458092212677,0.0001,4.551757574081421,28.121005549341316,4.21370792388916,13.199567317962646,6.317291259765625,3.8819658756256104
+2026-11-05T03:00:28Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,35,3,105,0.006470461376011372,0.002896874211728573,0.006953801494091749,0.005397097673267126,0.019380679354071617,0.04339449480175972,0.0001,4.577567100524902,20.971838946717316,4.20784854888916,13.199567317962646,6.317291259765625,0.17257003486156464
+2026-11-05T03:01:07Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,36,1,106,0.006960729602724314,0.0032099143136292696,0.008326738141477108,0.004777777474373579,0.020404072478413582,0.03957637771964073,0.0001,4.604785442352295,27.797169184632597,4.21370792388916,13.199567317962646,6.317295074462891,0.3869265615940094
+2026-11-05T03:01:12Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,36,2,107,0.009162246249616146,0.0028733815997838974,0.014910601079463959,0.012559023685753345,0.01990017108619213,0.06274174898862839,0.0001,4.682635068893433,27.335036388015617,4.21419620513916,13.199567317962646,6.317302703857422,0.697833240032196
+2026-11-05T03:01:17Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,36,3,108,0.007078883703798056,0.0029025920666754246,0.009396244771778584,0.0062500424683094025,0.019115250557661057,0.04537143185734749,0.0001,4.505893707275391,21.305429341352344,4.20736026763916,13.199567317962646,6.317314147949219,0.24839337170124054
+2026-11-05T03:01:56Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,37,1,109,0.020217763260006905,0.002673803362995386,0.05616578087210655,0.037825729697942734,0.018608858808875084,0.0891939103603363,0.0001,4.530768156051636,28.251280045974884,4.21419620513916,13.199567317962646,6.317314147949219,5.895182132720947
+2026-11-05T03:02:01Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,37,2,110,0.013528971932828426,0.002435058355331421,0.03549623116850853,0.022040992975234985,0.016377722844481468,0.05677291750907898,0.0001,4.784580707550049,26.752605468231838,4.21370792388916,13.199567317962646,6.317314147949219,1.671847939491272
+2026-11-05T03:02:05Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,37,3,111,0.004955140873789787,0.002207765821367502,0.006598646752536297,0.003183918772265315,0.01492322701960802,0.0256162341684103,0.0001,4.384855270385742,21.89353902929497,4.20821475982666,13.199567317962646,6.317317962646484,0.17068901658058167
+2026-11-05T03:02:45Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,38,1,112,0.005507960449904203,0.0020860431250184774,0.006718870718032122,0.006294739432632923,0.015052169561386108,0.042329091578722,0.0001,4.769389867782593,26.837814384738138,4.21370792388916,13.199567317962646,6.317317962646484,0.4342043995857239
+2026-11-05T03:02:49Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,38,2,113,0.00582362012937665,0.0021975692361593246,0.007395792752504349,0.0073803323321044445,0.014317098073661327,0.041808780282735825,0.0001,4.571729898452759,27.998154493623932,4.21419620513916,13.199567317962646,6.317352294921875,1.488032341003418
+2026-11-05T03:02:54Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,38,3,114,0.004494220484048128,0.0019903164356946945,0.005655961111187935,0.0035414707381278276,0.012403125874698162,0.026121987029910088,0.0001,4.508204460144043,21.29450890009826,4.20736026763916,13.199567317962646,6.317352294921875,0.46675992012023926
+2026-11-05T03:03:33Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,39,1,115,0.011992054060101509,0.0019914451986551285,0.02509084716439247,0.028782391920685768,0.01404609065502882,0.07711786776781082,0.0001,4.561222314834595,28.062653202344887,4.21419620513916,13.199567317962646,6.317356109619141,3.3818507194519043
+2026-11-05T03:03:38Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,39,2,116,0.00480043375864625,0.0018637379398569465,0.006048420909792185,0.004938909318298101,0.01286547165364027,0.035925060510635376,0.0001,4.768991231918335,26.840057734497403,4.21370792388916,13.199567317962646,6.317356109619141,0.368604838848114
+2026-11-05T03:03:42Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,39,3,117,0.019393030554056168,0.0020835930481553078,0.049099624156951904,0.050501544028520584,0.013543962500989437,0.09488000720739365,0.0001,4.431109666824341,21.665002046496554,4.20784854888916,13.199567317962646,6.317352294921875,4.5854291915893555
+2026-11-05T03:04:22Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,40,1,118,0.004243348725140095,0.0018372742924839258,0.004412771202623844,0.003772095311433077,0.012820491567254066,0.031674545258283615,0.0001,4.705322027206421,27.203239068420235,4.21370792388916,13.199567317962646,6.317352294921875,0.11168286949396133
+2026-11-05T03:04:27Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,40,2,119,0.004631279967725277,0.001685254741460085,0.006640085484832525,0.004888775292783976,0.011777923442423344,0.03290068730711937,0.0001,4.594045162200928,27.862155351271603,4.21419620513916,13.199567317962646,6.317352294921875,0.40780383348464966
+2026-11-05T03:04:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,40,3,120,0.003668498480692506,0.0018683280795812607,0.0036391704343259335,0.0018322281539440155,0.012461252510547638,0.01953316293656826,0.0001,4.653713226318359,20.628688389539512,4.20736026763916,13.199567317962646,6.317352294921875,0.1261381357908249

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/live_telemetry.json

@@ -1,5 +1,5 @@
 {
-  "gpu_utilization": 97.0,
+  "gpu_utilization": 40.0,
   "gpu_memory_mib": 15867.0,
-  "timestamp_unix": 1793847270.7125804
+  "timestamp_unix": 1793847880.6723254
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/state.json

@@ -2,69 +2,69 @@
   "arm": "spc_k0",
   "batch": 3,
   "batches": 3,
-  "best_acc": 0.9046296296296297,
-  "best_epoch": 15,
-  "checkpoint_identity": "f0bd9e7019176ff7d79817d2019285d4e877d52214972a13d757eddd67496fd0",
-  "completed_batches": 39,
-  "cpu_rss_gib": 6.317134857177734,
-  "epoch": 27,
+  "best_acc": 0.9112962962962963,
+  "best_epoch": 35,
+  "checkpoint_identity": "2cd5f72553ea3228ae87514795a3b29a1b307538510db93faaf433a12dcd5165",
+  "completed_batches": 12,
+  "cpu_rss_gib": 6.317352294921875,
+  "epoch": 40,
   "epoch_eta_seconds": 0.0,
   "epochs": 45,
-  "eta_seconds": 2.559461223391386,
-  "evaluated": 4992,
+  "eta_seconds": 24.304910004138947,
+  "evaluated": 1536,
   "evaluation_progress": {
     "arm": "spc_k0",
     "batch": 3,
     "batches": 3,
-    "checkpoint_identity": "f0bd9e7019176ff7d79817d2019285d4e877d52214972a13d757eddd67496fd0",
-    "completed_batches": 39,
-    "cpu_rss_gib": 6.317134857177734,
-    "epoch": 27,
+    "checkpoint_identity": "2cd5f72553ea3228ae87514795a3b29a1b307538510db93faaf433a12dcd5165",
+    "completed_batches": 12,
+    "cpu_rss_gib": 6.317352294921875,
+    "epoch": 40,
     "epoch_eta_seconds": 0.0,
     "epochs": 45,
-    "eta_seconds": 2.559461223391386,
-    "evaluated": 4992,
-    "global_step": 81,
-    "gpu_memory_gib": 4.20784854888916,
+    "eta_seconds": 24.304910004138947,
+    "evaluated": 1536,
+    "global_step": 120,
+    "gpu_memory_gib": 4.20736026763916,
     "gpu_peak_memory_gib": 13.199567317962646,
-    "gradient_norm": 0.9574737548828125,
-    "latest_chunk": "validation/epoch_027/chunks/00039.npz",
+    "gradient_norm": 0.1261381357908249,
+    "latest_chunk": "validation/epoch_040/chunks/00012.npz",
     "learning_rate": 0.0001,
-    "loss_epsilon": 0.04176785424351692,
-    "loss_real": 0.0054395608603954315,
-    "loss_reg_real": 0.02814420498907566,
-    "loss_reg_synthetic": 0.10715510696172714,
-    "loss_synthetic": 0.06417709589004517,
-    "loss_total": 0.025720637291669846,
+    "loss_epsilon": 0.0018322281539440155,
+    "loss_real": 0.0018683280795812607,
+    "loss_reg_real": 0.012461252510547638,
+    "loss_reg_synthetic": 0.01953316293656826,
+    "loss_synthetic": 0.0036391704343259335,
+    "loss_total": 0.003668498480692506,
     "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
     "operation": "official_forward_backward_optimizer",
     "phase": "VALIDATING",
     "pid": 10002,
     "recovery_segment_id": "20261105T023238Z",
     "samples": 5400,
-    "step_duration_seconds": 4.426917552947998,
-    "throughput": 159.4085490615029,
-    "timestamp_utc": "2026-11-05T02:53:54Z",
+    "step_duration_seconds": 4.653713226318359,
+    "throughput": 158.98022248763684,
+    "timestamp_utc": "2026-11-05T03:04:32Z",
     "total_batches": 43,
     "train_run_id": "experiment_E45_spc_k0",
-    "training_throughput": 21.68551793698329,
-    "updated_unix": 1793847266.289824,
-    "updated_utc": "2026-11-05T02:54:26Z"
+    "training_throughput": 20.628688389539512,
+    "updated_unix": 1793847882.2179294,
+    "updated_utc": "2026-11-05T03:04:42Z"
   },
   "experiment_id": "SPC_EUROSAT_HAIHT_PROMPTS_45E",
-  "global_step": 81,
-  "gpu_memory_gib": 4.20784854888916,
+  "global_step": 120,
+  "gpu_memory_gib": 4.20736026763916,
   "gpu_peak_memory_gib": 13.199567317962646,
-  "gradient_norm": 0.9574737548828125,
-  "heartbeat": "2026-11-05T02:54:27Z",
-  "latest_chunk": "validation/epoch_027/chunks/00039.npz",
+  "gradient_norm": 0.1261381357908249,
+  "heartbeat": "2026-11-05T03:04:42Z",
+  "latest_chunk": "validation/epoch_040/chunks/00012.npz",
   "learning_rate": 0.0001,
-  "loss_epsilon": 0.04176785424351692,
-  "loss_real": 0.0054395608603954315,
-  "loss_reg_real": 0.02814420498907566,
-  "loss_reg_synthetic": 0.10715510696172714,
-  "loss_synthetic": 0.06417709589004517,
-  "loss_total": 0.025720637291669846,
+  "loss_epsilon": 0.0018322281539440155,
+  "loss_real": 0.0018683280795812607,
+  "loss_reg_real": 0.012461252510547638,
+  "loss_reg_synthetic": 0.01953316293656826,
+  "loss_synthetic": 0.0036391704343259335,
+  "loss_total": 0.003668498480692506,
   "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
   "operation": "official_forward_backward_optimizer",
   "phase": "VALIDATING",
@@ -72,14 +72,14 @@
   "recovery_segment_id": "20261105T023238Z",
   "samples": 5400,
   "stage": "VALIDATING",
-  "step_duration_seconds": 4.426917552947998,
-  "throughput": 159.4085490615029,
-  "timestamp_utc": "2026-11-05T02:53:54Z",
+  "step_duration_seconds": 4.653713226318359,
+  "throughput": 158.98022248763684,
+  "timestamp_utc": "2026-11-05T03:04:32Z",
   "total_batches": 43,
   "total_epochs": 45,
   "train_run_id": "d45_e45_20261104T122347Z_E45",
-  "training_throughput": 21.68551793698329,
-  "updated_unix": 1793847266.289824,
-  "updated_utc": "2026-11-05T02:54:26Z",
+  "training_throughput": 20.628688389539512,
+  "updated_unix": 1793847882.2179294,
+  "updated_utc": "2026-11-05T03:04:42Z",
   "worker_pid": 10002
 }

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00001.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dce12db42717a5c2c00e01d3a888d5fdbe80cecd2eed7d63c3e59aeacb0633de
+size 8013

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00002.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6643bdc4213631c064099de19ac625b5b74cce6be25827d99a5eac1fb9492a3f
+size 8038

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00003.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:773058af38b168c2a098651935c12368cb50ccbdca29558f1be258c357430b27
+size 8031

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00004.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50a53f970ec7a423c870ecdebc9aea17abcab0b83fa89c70c917b7dddcf194e9
+size 8018

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00005.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5741a4a0994bf6202ada0a7af26f89f5814721f6337d65addd676211e7e0cbe8
+size 8021

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00006.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3c26b1032bba1c1e86aa8bf7032d4c47772602e164881996bfd0236c98921b
+size 8015

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00007.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d996ee797df9da24d9bc1b0f317c8c76c74efdb3979f3ea2928b17a82cfba902
+size 8006

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00008.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79876d0212db48b08e61dcbf83040086134cc1beccc5a778f3c2e8cba06257ce
+size 8028

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00009.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac11a53b427441f2185701b1047903b3f40e10faae50342625680b49d816fe7d
+size 8000

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00010.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5776d4d0a03891b4761a20dc13bb204bbdb6f6be98cd45e3b134b1c5cd004b5
+size 8028

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00011.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:365dad5aa1c20cf6fde8cd0b2253104a7e8c914ffc76723be8d0dd10ccf408ee
+size 8010

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00012.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:472124760e1a20ac2f0de8cfba20bd5a0785683ed38fb7dce3d4ced351d41796
+size 8035

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00013.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1a4ee99639928e8dd6a9753ba1f0b2c9fdd2dfbb19f5e8badd69143cd3c59fe
+size 8009

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00014.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8365b64e41b08b08a64aaa6bb6d6f56e9719970330f72bac8085b133d60570b
+size 8034

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00015.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb89e999ad83821325acf43cfe286f7d1cbd339415ed537e7884770f09dd1701
+size 8020

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00016.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d23b461b326ad776cf8254683557b7c04e515cc163189b5e5e790781170661a
+size 8002

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00017.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7e6881abbc489fe62ee6d81098d3319b1b97f7af26c4a2370ef2a9cb750aba6
+size 8048

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00018.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a01e331c6c6ec45458d5a98baa63b46315b11b9c83d852c018bacb1c3ff522c
+size 8029

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00019.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbb856c95b19f9a3da8bb6c189e737c17c64cc1a8b7ea014db17a35258bfd1d4
+size 8000

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00020.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df60c8b04bc971889ce266b5a785bb86852435dde10cf22fd93b150f94802b02
+size 8029

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00021.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f1a1a5f8ac1822fd6b09a155210d41272f74345bc07fd0bbffc0f68f2a2b393
+size 8027

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00022.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6d100c5893facbe937293cec3fce225beb1751bda3eb3783c0c87914ecd1973
+size 8029

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00023.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4b3f48ccb80fe0d3ec22afe23b6f1e4a9e6216c513aa755ad25ae4a9bf1e3b8
+size 8038

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00024.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f41246646e36841e09ae013ef085c10070a07006906449fb655e0f857ac9b35
+size 8024

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00025.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d868aa30ab8a6e98765570c0e82b1cce377a78dbe620ddb6ffd91de229724cbe
+size 8038

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00026.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c79f497a817178a6a130287ebe7b431ff8d841ba0d96199bd96944edb1a9f56
+size 8038

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00027.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7799fcbe5b5bce3b7326a911012d93ed21808cbf6bee3dd3429601a2b829f651
+size 8017

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00028.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ddb7b8a8c9387e68be09d045f26981131bb8b12f4afe3b09589c00648b0124b
+size 8025

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00029.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10d45adae650e0725ca3ec30c6a43723733246c82f830e49983460d77376b8b
+size 8032

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00030.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda650f0379e5eeff902a194234af1e8264e665a4dd1fde16fb2855f39124c80
+size 8022

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00031.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe4b5c4eff851d9774598e0364bd923a3574a827578e95d394503c183e2c186
+size 8021

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00032.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93c73fa5e66b16fa78d0d16cde3baccba0b597b48246be46c4a7da067d0393ff
+size 8031

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00033.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce33c5449b8b0f5bc87bee41746e8bf5a7d1167b09fa6b64953604f424163ad9
+size 8005

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00034.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f920bbd2ca769161a249caa82295954088cedfb0b01a8363d6d26b530955dd00
+size 8005

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00035.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e156f1f124c25d4222d7e1d6e2e67506b123a8029f7676da027a227c65d55e2
+size 8029

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00036.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29bee4c3b39476655da5586c6a3ef2d4d94e00e047e97343309df48b715b06e2
+size 8027

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00037.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:225011aee13bb1c6127ba45f1465a565dcf415510ac99575a7ef064ce2fac81a
+size 8023

experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_028/chunks/00038.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24983808bc0d14a2d89ed204a2297c4b5138ac5bfe1ff01ab37a483b01103c95
+size 8026