Verified D45/E45 source, measured progress and independent recovery checkpoints
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/BEST.json +5 -5
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/CURRENT.json +5 -5
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/PREVIOUS.json +6 -6
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/finetuning/progress.json +7 -7
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/epoch_metrics.csv +13 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/events.jsonl +65 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/status.json +23 -23
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/train_steps.csv +39 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/live_telemetry.json +2 -2
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/worker_status.json +23 -23
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/state.json +49 -49
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00041.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00042.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00043.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/metrics.json +197 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/predictions.jsonl +0 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/progress.json +6 -6
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/recovery_journal.json +5 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00001.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00002.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00003.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00004.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00005.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00006.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00007.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00008.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00009.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00010.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00011.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00012.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00013.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00014.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00015.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00016.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00017.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00018.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00019.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00020.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00021.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00022.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00023.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00024.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00025.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00026.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00027.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00028.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00029.npz +3 -0
- experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00030.npz +3 -0
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/BEST.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
"bytes": 67597675,
|
| 4 |
-
"epoch_index":
|
| 5 |
-
"global_step":
|
| 6 |
-
"path": "checkpoints/history/
|
| 7 |
"remote_verified": false,
|
| 8 |
-
"sha256": "
|
| 9 |
-
"timestamp_utc": "2026-11-05T02:
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
"bytes": 67597675,
|
| 4 |
+
"epoch_index": 14,
|
| 5 |
+
"global_step": 42,
|
| 6 |
+
"path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt",
|
| 7 |
"remote_verified": false,
|
| 8 |
+
"sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92",
|
| 9 |
+
"timestamp_utc": "2026-11-05T02:43:54Z"
|
| 10 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/CURRENT.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
"bytes": 67597675,
|
| 4 |
-
"epoch_index":
|
| 5 |
-
"global_step":
|
| 6 |
-
"path": "checkpoints/history/
|
| 7 |
"remote_verified": false,
|
| 8 |
-
"sha256": "
|
| 9 |
-
"timestamp_utc": "2026-11-05T02:
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
"bytes": 67597675,
|
| 4 |
+
"epoch_index": 14,
|
| 5 |
+
"global_step": 42,
|
| 6 |
+
"path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt",
|
| 7 |
"remote_verified": false,
|
| 8 |
+
"sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92",
|
| 9 |
+
"timestamp_utc": "2026-11-05T02:43:54Z"
|
| 10 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/PREVIOUS.json
CHANGED
|
@@ -1,10 +1,10 @@
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
-
"bytes":
|
| 4 |
-
"epoch_index":
|
| 5 |
-
"global_step":
|
| 6 |
-
"path": "checkpoints/history/
|
| 7 |
"remote_verified": false,
|
| 8 |
-
"sha256": "
|
| 9 |
-
"timestamp_utc": "2026-11-05T02:
|
| 10 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"batch_next": 0,
|
| 3 |
+
"bytes": 67597675,
|
| 4 |
+
"epoch_index": 13,
|
| 5 |
+
"global_step": 39,
|
| 6 |
+
"path": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt",
|
| 7 |
"remote_verified": false,
|
| 8 |
+
"sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c",
|
| 9 |
+
"timestamp_utc": "2026-11-05T02:43:05Z"
|
| 10 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c
|
| 3 |
+
size 67597675
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92
|
| 3 |
+
size 67597675
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/finetuning/progress.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
{
|
| 2 |
-
"best_epoch":
|
| 3 |
-
"best_validation_accuracy": 0.
|
| 4 |
-
"best_validation_macro_f1": 0.
|
| 5 |
-
"completed_epochs":
|
| 6 |
-
"global_step":
|
| 7 |
-
"latest_validation_accuracy": 0.
|
| 8 |
-
"latest_validation_macro_f1": 0.
|
| 9 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"best_epoch": 14,
|
| 3 |
+
"best_validation_accuracy": 0.9007407407407407,
|
| 4 |
+
"best_validation_macro_f1": 0.8976430827821179,
|
| 5 |
+
"completed_epochs": 14,
|
| 6 |
+
"global_step": 42,
|
| 7 |
+
"latest_validation_accuracy": 0.9007407407407407,
|
| 8 |
+
"latest_validation_macro_f1": 0.8976430827821179
|
| 9 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/epoch_metrics.csv
CHANGED
|
@@ -1,2 +1,15 @@
|
|
| 1 |
timestamp_utc,epoch,global_step,train_loss_mean,validation_accuracy,validation_macro_f1,best_validation_accuracy,best_epoch,duration_seconds,checkpoint,checkpoint_sha256
|
| 2 |
2026-11-05T02:33:27Z,1,3,2.0959319273630777,0.5385185185185185,0.4921045345099603,0.5385185185185185,1,49.75665736198425,checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt,1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
timestamp_utc,epoch,global_step,train_loss_mean,validation_accuracy,validation_macro_f1,best_validation_accuracy,best_epoch,duration_seconds,checkpoint,checkpoint_sha256
|
| 2 |
2026-11-05T02:33:27Z,1,3,2.0959319273630777,0.5385185185185185,0.4921045345099603,0.5385185185185185,1,49.75665736198425,checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt,1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1
|
| 3 |
+
2026-11-05T02:34:17Z,2,6,1.6399901310602825,0.6287037037037037,0.5767701775459999,0.6287037037037037,2,49.57090616226196,checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt,1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca
|
| 4 |
+
2026-11-05T02:35:05Z,3,9,1.3544566631317139,0.6751851851851852,0.6327180638698691,0.6751851851851852,3,48.011171102523804,checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt,5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a
|
| 5 |
+
2026-11-05T02:35:53Z,4,12,1.0348247090975444,0.7431481481481481,0.716856644247706,0.7431481481481481,4,47.63766622543335,checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt,65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973
|
| 6 |
+
2026-11-05T02:36:40Z,5,15,0.8252936999003092,0.7922222222222223,0.7708581649718522,0.7922222222222223,5,47.327789306640625,checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt,2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f
|
| 7 |
+
2026-11-05T02:37:27Z,6,18,0.6310902635256449,0.8362962962962963,0.8248050745115829,0.8362962962962963,6,47.18750858306885,checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt,9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482
|
| 8 |
+
2026-11-05T02:38:15Z,7,21,0.4793592691421509,0.8651851851851852,0.8583025469404868,0.8651851851851852,7,47.810460567474365,checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt,f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd
|
| 9 |
+
2026-11-05T02:39:02Z,8,24,0.35045451919237774,0.8751851851851852,0.8706107374650793,0.8751851851851852,8,46.78648495674133,checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt,73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3
|
| 10 |
+
2026-11-05T02:39:49Z,9,27,0.24601457516352335,0.8855555555555555,0.8808243497096742,0.8855555555555555,9,47.03523898124695,checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt,7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd
|
| 11 |
+
2026-11-05T02:40:38Z,10,30,0.20679345230261484,0.8822222222222222,0.8777383631501999,0.8855555555555555,9,48.99299097061157,checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt,317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2
|
| 12 |
+
2026-11-05T02:41:27Z,11,33,0.15470425287882486,0.8905555555555555,0.8871090790401457,0.8905555555555555,11,49.47192049026489,checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt,a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004
|
| 13 |
+
2026-11-05T02:42:17Z,12,36,0.12127700696388881,0.8931481481481481,0.8896603607977387,0.8931481481481481,12,49.385294914245605,checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt,31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7
|
| 14 |
+
2026-11-05T02:43:05Z,13,39,0.0904170572757721,0.8987037037037037,0.8954650689040694,0.8987037037037037,13,48.701061725616455,checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt,48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c
|
| 15 |
+
2026-11-05T02:43:54Z,14,42,0.08223462104797363,0.9007407407407407,0.8976430827821179,0.9007407407407407,14,48.82230854034424,checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt,ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/events.jsonl
CHANGED
|
@@ -9,3 +9,68 @@
|
|
| 9 |
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.339874267578125, "epoch": 2, "global_step": 4, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.7395637035369873, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6463867425918579, "loss_real": 1.3266286849975586, "loss_reg_real": 2.305392026901245, "loss_reg_synthetic": 2.1597659587860107, "loss_synthetic": 1.336653470993042, "loss_total": 1.7940089702606201, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6243250370025635, "timestamp_utc": "2026-11-05T02:33:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.679715196440473}
|
| 10 |
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3444061279296875, "epoch": 2, "global_step": 5, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.4725263118743896, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6717017292976379, "loss_real": 1.2274569272994995, "loss_reg_real": 2.1863515377044678, "loss_reg_synthetic": 2.067314863204956, "loss_synthetic": 1.4349440336227417, "loss_total": 1.710416316986084, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.548011302947998, "timestamp_utc": "2026-11-05T02:33:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.144169280544894}
|
| 11 |
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.345935821533203, "epoch": 2, "global_step": 6, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 4.890103340148926, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5022993087768555, "loss_real": 0.9928121566772461, "loss_reg_real": 1.8100521564483643, "loss_reg_synthetic": 1.9769306182861328, "loss_synthetic": 1.3028115034103394, "loss_total": 1.415545105934143, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.50909423828125, "timestamp_utc": "2026-11-05T02:33:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.29030686140477}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.339874267578125, "epoch": 2, "global_step": 4, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.7395637035369873, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6463867425918579, "loss_real": 1.3266286849975586, "loss_reg_real": 2.305392026901245, "loss_reg_synthetic": 2.1597659587860107, "loss_synthetic": 1.336653470993042, "loss_total": 1.7940089702606201, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6243250370025635, "timestamp_utc": "2026-11-05T02:33:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.679715196440473}
|
| 10 |
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3444061279296875, "epoch": 2, "global_step": 5, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.4725263118743896, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6717017292976379, "loss_real": 1.2274569272994995, "loss_reg_real": 2.1863515377044678, "loss_reg_synthetic": 2.067314863204956, "loss_synthetic": 1.4349440336227417, "loss_total": 1.710416316986084, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.548011302947998, "timestamp_utc": "2026-11-05T02:33:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.144169280544894}
|
| 11 |
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.345935821533203, "epoch": 2, "global_step": 6, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 4.890103340148926, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5022993087768555, "loss_real": 0.9928121566772461, "loss_reg_real": 1.8100521564483643, "loss_reg_synthetic": 1.9769306182861328, "loss_synthetic": 1.3028115034103394, "loss_total": 1.415545105934143, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.50909423828125, "timestamp_utc": "2026-11-05T02:33:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.29030686140477}
|
| 12 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 2, "global_step": 6, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt", "remote_verified": false, "sha256": "1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca", "timestamp_utc": "2026-11-05T02:34:17Z"}
|
| 13 |
+
{"arm": "spc_k0", "best_epoch": 2, "best_validation_accuracy": 0.6287037037037037, "checkpoint": "checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt", "checkpoint_sha256": "1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca", "duration_seconds": 49.57090616226196, "epoch": 2, "global_step": 6, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:34:17Z", "train_loss_mean": 1.6399901310602825, "validation_accuracy": 0.6287037037037037, "validation_macro_f1": 0.5767701775459999}
|
| 14 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348064422607422, "epoch": 3, "global_step": 7, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 3.2753567695617676, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.532727837562561, "loss_real": 0.8864139318466187, "loss_reg_real": 1.7116870880126953, "loss_reg_synthetic": 1.9229612350463867, "loss_synthetic": 1.1026084423065186, "loss_total": 1.2671351432800293, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6091248989105225, "timestamp_utc": "2026-11-05T02:34:22Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.770998358116934}
|
| 15 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3471527099609375, "epoch": 3, "global_step": 8, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 5.345407485961914, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5581454634666443, "loss_real": 0.9600490927696228, "loss_reg_real": 1.8062270879745483, "loss_reg_synthetic": 1.8906059265136719, "loss_synthetic": 1.2284905910491943, "loss_total": 1.371622920036316, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.7241575717926025, "timestamp_utc": "2026-11-05T02:34:26Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.094777863522836}
|
| 16 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.348278045654297, "epoch": 3, "global_step": 9, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 8.87842082977295, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.7060779929161072, "loss_real": 1.022498607635498, "loss_reg_real": 1.7766468524932861, "loss_reg_synthetic": 1.8551563024520874, "loss_synthetic": 1.1166820526123047, "loss_total": 1.4246119260787964, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.492733001708984, "timestamp_utc": "2026-11-05T02:34:31Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.367840012634336}
|
| 17 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 3, "global_step": 9, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt", "remote_verified": false, "sha256": "5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a", "timestamp_utc": "2026-11-05T02:35:05Z"}
|
| 18 |
+
{"arm": "spc_k0", "best_epoch": 3, "best_validation_accuracy": 0.6751851851851852, "checkpoint": "checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt", "checkpoint_sha256": "5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a", "duration_seconds": 48.011171102523804, "epoch": 3, "global_step": 9, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:35:05Z", "train_loss_mean": 1.3544566631317139, "validation_accuracy": 0.6751851851851852, "validation_macro_f1": 0.6327180638698691}
|
| 19 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348358154296875, "epoch": 4, "global_step": 10, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 5.862489223480225, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5491296648979187, "loss_real": 0.8255103826522827, "loss_reg_real": 1.4897865056991577, "loss_reg_synthetic": 1.6701234579086304, "loss_synthetic": 1.0721659660339355, "loss_total": 1.1878503561019897, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.607059717178345, "timestamp_utc": "2026-11-05T02:35:10Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.783447113291448}
|
| 20 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.348392486572266, "epoch": 4, "global_step": 11, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 3.9853928089141846, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5843984484672546, "loss_real": 0.6818434000015259, "loss_reg_real": 1.5178686380386353, "loss_reg_synthetic": 1.6483911275863647, "loss_synthetic": 1.025932788848877, "loss_total": 1.0391523838043213, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.588322877883911, "timestamp_utc": "2026-11-05T02:35:14Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.896903379875553}
|
| 21 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.3484649658203125, "epoch": 4, "global_step": 12, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 4.246176242828369, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5330894589424133, "loss_real": 0.5594723224639893, "loss_reg_real": 1.2257851362228394, "loss_reg_synthetic": 1.715860366821289, "loss_synthetic": 0.9067077040672302, "loss_total": 0.877471387386322, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.472378730773926, "timestamp_utc": "2026-11-05T02:35:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.465087323538814}
|
| 22 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 4, "global_step": 12, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt", "remote_verified": false, "sha256": "65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973", "timestamp_utc": "2026-11-05T02:35:53Z"}
|
| 23 |
+
{"arm": "spc_k0", "best_epoch": 4, "best_validation_accuracy": 0.7431481481481481, "checkpoint": "checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt", "checkpoint_sha256": "65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973", "duration_seconds": 47.63766622543335, "epoch": 4, "global_step": 12, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:35:53Z", "train_loss_mean": 1.0348247090975444, "validation_accuracy": 0.7431481481481481, "validation_macro_f1": 0.716856644247706}
|
| 24 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348529815673828, "epoch": 5, "global_step": 13, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.1963372230529785, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5366214513778687, "loss_real": 0.6004937887191772, "loss_reg_real": 1.2908483743667603, "loss_reg_synthetic": 1.5194084644317627, "loss_synthetic": 0.875447690486908, "loss_total": 0.9112675786018372, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.4592626094818115, "timestamp_utc": "2026-11-05T02:35:57Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.70429737146031}
|
| 25 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.348590850830078, "epoch": 5, "global_step": 14, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.359305381774902, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5021079182624817, "loss_real": 0.46975645422935486, "loss_reg_real": 1.1514276266098022, "loss_reg_synthetic": 1.4126462936401367, "loss_synthetic": 0.8143174648284912, "loss_total": 0.7571407556533813, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.563748598098755, "timestamp_utc": "2026-11-05T02:36:02Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.047118996283988}
|
| 26 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.348659515380859, "epoch": 5, "global_step": 15, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.185586929321289, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5958570837974548, "loss_real": 0.5067290663719177, "loss_reg_real": 1.0209497213363647, "loss_reg_synthetic": 1.5450220108032227, "loss_synthetic": 0.8470979928970337, "loss_total": 0.8074727654457092, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.366473197937012, "timestamp_utc": "2026-11-05T02:36:06Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.985706919111802}
|
| 27 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 5, "global_step": 15, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt", "remote_verified": false, "sha256": "2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f", "timestamp_utc": "2026-11-05T02:36:40Z"}
|
| 28 |
+
{"arm": "spc_k0", "best_epoch": 5, "best_validation_accuracy": 0.7922222222222223, "checkpoint": "checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt", "checkpoint_sha256": "2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f", "duration_seconds": 47.327789306640625, "epoch": 5, "global_step": 15, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:36:40Z", "train_loss_mean": 0.8252936999003092, "validation_accuracy": 0.7922222222222223, "validation_macro_f1": 0.7708581649718522}
|
| 29 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352695465087891, "epoch": 6, "global_step": 16, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.249288082122803, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.47728967666625977, "loss_real": 0.48116928339004517, "loss_reg_real": 1.0561646223068237, "loss_reg_synthetic": 1.4124089479446411, "loss_synthetic": 0.7735217809677124, "loss_total": 0.7540974020957947, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.513085842132568, "timestamp_utc": "2026-11-05T02:36:45Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.361968834059706}
|
| 30 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352790832519531, "epoch": 6, "global_step": 17, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.7728734016418457, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.40542393922805786, "loss_real": 0.31128787994384766, "loss_reg_real": 0.8898597955703735, "loss_reg_synthetic": 1.2429455518722534, "loss_synthetic": 0.6802616119384766, "loss_total": 0.5483358502388, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.580164432525635, "timestamp_utc": "2026-11-05T02:36:49Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.94659490629185}
|
| 31 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352806091308594, "epoch": 6, "global_step": 18, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 6.730944633483887, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.4384499490261078, "loss_real": 0.32776927947998047, "loss_reg_real": 0.7986207008361816, "loss_reg_synthetic": 1.2806344032287598, "loss_synthetic": 0.8083286285400391, "loss_total": 0.5908375382423401, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.276374340057373, "timestamp_utc": "2026-11-05T02:36:53Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 22.448923402414774}
|
| 32 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 6, "global_step": 18, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt", "remote_verified": false, "sha256": "9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482", "timestamp_utc": "2026-11-05T02:37:27Z"}
|
| 33 |
+
{"arm": "spc_k0", "best_epoch": 6, "best_validation_accuracy": 0.8362962962962963, "checkpoint": "checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt", "checkpoint_sha256": "9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482", "duration_seconds": 47.18750858306885, "epoch": 6, "global_step": 18, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:37:27Z", "train_loss_mean": 0.6310902635256449, "validation_accuracy": 0.8362962962962963, "validation_macro_f1": 0.8248050745115829}
|
| 34 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352817535400391, "epoch": 7, "global_step": 19, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.4259095191955566, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3979944884777069, "loss_real": 0.27780306339263916, "loss_reg_real": 0.7724456191062927, "loss_reg_synthetic": 1.2027769088745117, "loss_synthetic": 0.687814474105835, "loss_total": 0.5101187825202942, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.736155033111572, "timestamp_utc": "2026-11-05T02:37:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.026142325393053}
|
| 35 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352817535400391, "epoch": 7, "global_step": 20, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.5623104572296143, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.280882865190506, "loss_real": 0.20820730924606323, "loss_reg_real": 0.7042987942695618, "loss_reg_synthetic": 1.0282939672470093, "loss_synthetic": 0.46521008014678955, "loss_total": 0.37807542085647583, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.497614622116089, "timestamp_utc": "2026-11-05T02:37:36Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.459530385415082}
|
| 36 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.35284423828125, "epoch": 7, "global_step": 21, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.640958309173584, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3802229166030884, "loss_real": 0.3616764545440674, "loss_reg_real": 0.5518221259117126, "loss_reg_synthetic": 1.0541349649429321, "loss_synthetic": 0.5351462960243225, "loss_total": 0.5498836040496826, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.428439140319824, "timestamp_utc": "2026-11-05T02:37:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.678066912096444}
|
| 37 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 7, "global_step": 21, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt", "remote_verified": false, "sha256": "f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd", "timestamp_utc": "2026-11-05T02:38:15Z"}
|
| 38 |
+
{"arm": "spc_k0", "best_epoch": 7, "best_validation_accuracy": 0.8651851851851852, "checkpoint": "checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt", "checkpoint_sha256": "f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd", "duration_seconds": 47.810460567474365, "epoch": 7, "global_step": 21, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:38:15Z", "train_loss_mean": 0.4793592691421509, "validation_accuracy": 0.8651851851851852, "validation_macro_f1": 0.8583025469404868}
|
| 39 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352836608886719, "epoch": 8, "global_step": 22, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.94069766998291, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.38374873995780945, "loss_real": 0.19556286931037903, "loss_reg_real": 0.5717012286186218, "loss_reg_synthetic": 1.051354169845581, "loss_synthetic": 0.4724317193031311, "loss_total": 0.37231922149658203, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.457226276397705, "timestamp_utc": "2026-11-05T02:38:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.717411246944497}
|
| 40 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352851867675781, "epoch": 8, "global_step": 23, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.106137752532959, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.35716506838798523, "loss_real": 0.144283726811409, "loss_reg_real": 0.4893544316291809, "loss_reg_synthetic": 0.9479215741157532, "loss_synthetic": 0.4486972689628601, "loss_total": 0.30827227234840393, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.544239521026611, "timestamp_utc": "2026-11-05T02:38:24Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.167529331966836}
|
| 41 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352886199951172, "epoch": 8, "global_step": 24, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.545416355133057, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.40896984934806824, "loss_real": 0.1848861277103424, "loss_reg_real": 0.44575411081314087, "loss_reg_synthetic": 1.0729613304138184, "loss_synthetic": 0.5284978151321411, "loss_total": 0.3707720637321472, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.430412530899048, "timestamp_utc": "2026-11-05T02:38:28Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.668411085980534}
|
| 42 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 8, "global_step": 24, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt", "remote_verified": false, "sha256": "73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3", "timestamp_utc": "2026-11-05T02:39:02Z"}
|
| 43 |
+
{"arm": "spc_k0", "best_epoch": 8, "best_validation_accuracy": 0.8751851851851852, "checkpoint": "checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt", "checkpoint_sha256": "73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3", "duration_seconds": 46.78648495674133, "epoch": 8, "global_step": 24, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:39:02Z", "train_loss_mean": 0.35045451919237774, "validation_accuracy": 0.8751851851851852, "validation_macro_f1": 0.8706107374650793}
|
| 44 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.3529052734375, "epoch": 9, "global_step": 25, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.5497570037841797, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.28275266289711, "loss_real": 0.10441966354846954, "loss_reg_real": 0.378287136554718, "loss_reg_synthetic": 0.8491285443305969, "loss_synthetic": 0.3608815670013428, "loss_total": 0.23698531091213226, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.535761594772339, "timestamp_utc": "2026-11-05T02:39:06Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.220178094793503}
|
| 45 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352931976318359, "epoch": 9, "global_step": 26, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 6.295091152191162, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3139715790748596, "loss_real": 0.13280969858169556, "loss_reg_real": 0.3783737123012543, "loss_reg_synthetic": 0.8116950392723083, "loss_synthetic": 0.398764044046402, "loss_total": 0.27532851696014404, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.540703773498535, "timestamp_utc": "2026-11-05T02:39:11Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.189462776026495}
|
| 46 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352973937988281, "epoch": 9, "global_step": 27, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.602907180786133, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.267505943775177, "loss_real": 0.10692095756530762, "loss_reg_real": 0.3405652940273285, "loss_reg_synthetic": 0.7080139517784119, "loss_synthetic": 0.3213772773742676, "loss_total": 0.22572989761829376, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.302645921707153, "timestamp_utc": "2026-11-05T02:39:15Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 22.311852229269714}
|
| 47 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 9, "global_step": 27, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt", "remote_verified": false, "sha256": "7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd", "timestamp_utc": "2026-11-05T02:39:49Z"}
|
| 48 |
+
{"arm": "spc_k0", "best_epoch": 9, "best_validation_accuracy": 0.8855555555555555, "checkpoint": "checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt", "checkpoint_sha256": "7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd", "duration_seconds": 47.03523898124695, "epoch": 9, "global_step": 27, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:39:49Z", "train_loss_mean": 0.24601457516352335, "validation_accuracy": 0.8855555555555555, "validation_macro_f1": 0.8808243497096742}
|
| 49 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352973937988281, "epoch": 10, "global_step": 28, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.6516027450561523, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.26639851927757263, "loss_real": 0.07133440673351288, "loss_reg_real": 0.26976266503334045, "loss_reg_synthetic": 0.7569214105606079, "loss_synthetic": 0.29729902744293213, "loss_total": 0.18336300551891327, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.613707780838013, "timestamp_utc": "2026-11-05T02:39:53Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.74341290786099}
|
| 50 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352993011474609, "epoch": 10, "global_step": 29, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.791995525360107, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.38612720370292664, "loss_real": 0.10022659599781036, "loss_reg_real": 0.32152873277664185, "loss_reg_synthetic": 0.8502084612846375, "loss_synthetic": 0.44544264674186707, "loss_total": 0.2577931582927704, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.586455821990967, "timestamp_utc": "2026-11-05T02:39:58Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.90825966016513}
|
| 51 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.353000640869141, "epoch": 10, "global_step": 30, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.241050958633423, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3099075257778168, "loss_real": 0.05060635507106781, "loss_reg_real": 0.23522202670574188, "loss_reg_synthetic": 0.75429767370224, "loss_synthetic": 0.36566129326820374, "loss_total": 0.1792241930961609, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.429757595062256, "timestamp_utc": "2026-11-05T02:40:02Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.671614741856054}
|
| 52 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 10, "global_step": 30, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt", "remote_verified": false, "sha256": "317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2", "timestamp_utc": "2026-11-05T02:40:38Z"}
|
| 53 |
+
{"arm": "spc_k0", "best_epoch": 9, "best_validation_accuracy": 0.8855555555555555, "checkpoint": "checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt", "checkpoint_sha256": "317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2", "duration_seconds": 48.99299097061157, "epoch": 10, "global_step": 30, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:40:38Z", "train_loss_mean": 0.20679345230261484, "validation_accuracy": 0.8822222222222222, "validation_macro_f1": 0.8777383631501999}
|
| 54 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313625335693359, "epoch": 11, "global_step": 31, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.2985382080078125, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2581752836704254, "loss_real": 0.06394229829311371, "loss_reg_real": 0.23884811997413635, "loss_reg_synthetic": 0.6717081069946289, "loss_synthetic": 0.3720165193080902, "loss_total": 0.1871512085199356, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6319260597229, "timestamp_utc": "2026-11-05T02:40:42Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.634292592239145}
|
| 55 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313652038574219, "epoch": 11, "global_step": 32, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.4699645042419434, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.16946427524089813, "loss_real": 0.05497533455491066, "loss_reg_real": 0.21618342399597168, "loss_reg_synthetic": 0.5425635576248169, "loss_synthetic": 0.2134230136871338, "loss_total": 0.13410496711730957, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.8092944622039795, "timestamp_utc": "2026-11-05T02:40:47Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.615130557287774}
|
| 56 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.31365966796875, "epoch": 11, "global_step": 33, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.234697103500366, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.19439589977264404, "loss_real": 0.045638084411621094, "loss_reg_real": 0.1863006353378296, "loss_reg_synthetic": 0.6746764183044434, "loss_synthetic": 0.28416675329208374, "loss_total": 0.14285658299922943, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.476435422897339, "timestamp_utc": "2026-11-05T02:40:52Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.445634959671715}
|
| 57 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 11, "global_step": 33, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt", "remote_verified": false, "sha256": "a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004", "timestamp_utc": "2026-11-05T02:41:27Z"}
|
| 58 |
+
{"arm": "spc_k0", "best_epoch": 11, "best_validation_accuracy": 0.8905555555555555, "checkpoint": "checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt", "checkpoint_sha256": "a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004", "duration_seconds": 49.47192049026489, "epoch": 11, "global_step": 33, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:41:27Z", "train_loss_mean": 0.15470425287882486, "validation_accuracy": 0.8905555555555555, "validation_macro_f1": 0.8871090790401457}
|
| 59 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313663482666016, "epoch": 12, "global_step": 34, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.473418235778809, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2995773255825043, "loss_real": 0.041587233543395996, "loss_reg_real": 0.1665458232164383, "loss_reg_synthetic": 0.7011368870735168, "loss_synthetic": 0.2889729142189026, "loss_total": 0.15002411603927612, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.786452054977417, "timestamp_utc": "2026-11-05T02:41:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.742146067647997}
|
| 60 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.31365966796875, "epoch": 12, "global_step": 35, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.8804813623428345, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.1297117918729782, "loss_real": 0.03432588279247284, "loss_reg_real": 0.16799309849739075, "loss_reg_synthetic": 0.44184446334838867, "loss_synthetic": 0.16249723732471466, "loss_total": 0.09535311907529831, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.679177761077881, "timestamp_utc": "2026-11-05T02:41:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.35523344821897}
|
| 61 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.3136749267578125, "epoch": 12, "global_step": 36, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.603003978729248, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.1828250139951706, "loss_real": 0.041674211621284485, "loss_reg_real": 0.14130644500255585, "loss_reg_synthetic": 0.5113574862480164, "loss_synthetic": 0.21308833360671997, "loss_total": 0.11845378577709198, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.5488574504852295, "timestamp_utc": "2026-11-05T02:41:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.10420057013649}
|
| 62 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 12, "global_step": 36, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt", "remote_verified": false, "sha256": "31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7", "timestamp_utc": "2026-11-05T02:42:17Z"}
|
| 63 |
+
{"arm": "spc_k0", "best_epoch": 12, "best_validation_accuracy": 0.8931481481481481, "checkpoint": "checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt", "checkpoint_sha256": "31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7", "duration_seconds": 49.385294914245605, "epoch": 12, "global_step": 36, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:42:17Z", "train_loss_mean": 0.12127700696388881, "validation_accuracy": 0.8931481481481481, "validation_macro_f1": 0.8896603607977387}
|
| 64 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313690185546875, "epoch": 13, "global_step": 37, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.5368391275405884, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.10143797844648361, "loss_real": 0.03273775428533554, "loss_reg_real": 0.13649021089076996, "loss_reg_synthetic": 0.3714805841445923, "loss_synthetic": 0.1622830331325531, "loss_total": 0.08822737634181976, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.575104475021362, "timestamp_utc": "2026-11-05T02:42:21Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.97750318027488}
|
| 65 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313724517822266, "epoch": 13, "global_step": 38, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.45637583732605, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.20935462415218353, "loss_real": 0.0280852522701025, "loss_reg_real": 0.12176745384931564, "loss_reg_synthetic": 0.5415651798248291, "loss_synthetic": 0.21965403854846954, "loss_total": 0.10865352302789688, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.667818307876587, "timestamp_utc": "2026-11-05T02:42:26Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.421804268604408}
|
| 66 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313724517822266, "epoch": 13, "global_step": 39, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.1801849603652954, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.11240734905004501, "loss_real": 0.02421887405216694, "loss_reg_real": 0.1274520456790924, "loss_reg_synthetic": 0.36493992805480957, "loss_synthetic": 0.13256892561912537, "loss_total": 0.07437027245759964, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.467528581619263, "timestamp_utc": "2026-11-05T02:42:30Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.488390783882718}
|
| 67 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 13, "global_step": 39, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt", "remote_verified": false, "sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c", "timestamp_utc": "2026-11-05T02:43:05Z"}
|
| 68 |
+
{"arm": "spc_k0", "best_epoch": 13, "best_validation_accuracy": 0.8987037037037037, "checkpoint": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt", "checkpoint_sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c", "duration_seconds": 48.701061725616455, "epoch": 13, "global_step": 39, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:43:05Z", "train_loss_mean": 0.0904170572757721, "validation_accuracy": 0.8987037037037037, "validation_macro_f1": 0.8954650689040694}
|
| 69 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313732147216797, "epoch": 14, "global_step": 40, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.9312450885772705, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2756049335002899, "loss_real": 0.017928674817085266, "loss_reg_real": 0.10229283571243286, "loss_reg_synthetic": 0.6242587566375732, "loss_synthetic": 0.24642586708068848, "loss_total": 0.11135122925043106, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.674060106277466, "timestamp_utc": "2026-11-05T02:43:10Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.38518484777088}
|
| 70 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313728332519531, "epoch": 14, "global_step": 41, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.085524082183838, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.11426927894353867, "loss_real": 0.020908314734697342, "loss_reg_real": 0.10161476582288742, "loss_reg_synthetic": 0.3838481903076172, "loss_synthetic": 0.14373445510864258, "loss_total": 0.07282369583845139, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.7962212562561035, "timestamp_utc": "2026-11-05T02:43:15Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.687676226996228}
|
| 71 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313732147216797, "epoch": 14, "global_step": 42, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.676707148551941, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.06848233938217163, "loss_real": 0.02631022036075592, "loss_reg_real": 0.09765395522117615, "loss_reg_synthetic": 0.27334070205688477, "loss_synthetic": 0.09998758137226105, "loss_total": 0.06252893805503845, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.421590089797974, "timestamp_utc": "2026-11-05T02:43:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.71164627438052}
|
| 72 |
+
{"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 14, "global_step": 42, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt", "remote_verified": false, "sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92", "timestamp_utc": "2026-11-05T02:43:54Z"}
|
| 73 |
+
{"arm": "spc_k0", "best_epoch": 14, "best_validation_accuracy": 0.9007407407407407, "checkpoint": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt", "checkpoint_sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92", "duration_seconds": 48.82230854034424, "epoch": 14, "global_step": 42, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:43:54Z", "train_loss_mean": 0.08223462104797363, "validation_accuracy": 0.9007407407407407, "validation_macro_f1": 0.8976430827821179}
|
| 74 |
+
{"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313716888427734, "epoch": 15, "global_step": 43, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.288538455963135, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.16103018820285797, "loss_real": 0.01848646253347397, "loss_reg_real": 0.08879290521144867, "loss_reg_synthetic": 0.37610241770744324, "loss_synthetic": 0.19105781614780426, "loss_total": 0.08387480676174164, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.769833326339722, "timestamp_utc": "2026-11-05T02:43:59Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.83531923288916}
|
| 75 |
+
{"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313720703125, "epoch": 15, "global_step": 44, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.8296756744384766, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.18699847161769867, "loss_real": 0.01577068120241165, "loss_reg_real": 0.08303209394216537, "loss_reg_synthetic": 0.38371822237968445, "loss_synthetic": 0.1664869785308838, "loss_total": 0.07876357436180115, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.5843305587768555, "timestamp_utc": "2026-11-05T02:44:04Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.92119773189996}
|
| 76 |
+
{"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313755035400391, "epoch": 15, "global_step": 45, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.7890217304229736, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.13382680714130402, "loss_real": 0.01956470124423504, "loss_reg_real": 0.1038849875330925, "loss_reg_synthetic": 0.36285677552223206, "loss_synthetic": 0.15807963907718658, "loss_total": 0.07597584277391434, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.581945180892944, "timestamp_utc": "2026-11-05T02:44:08Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 20.951800209292596}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/status.json
CHANGED
|
@@ -2,38 +2,38 @@
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
-
"checkpoint_identity": "
|
| 6 |
-
"completed_batches":
|
| 7 |
-
"cpu_rss_gib": 6.
|
| 8 |
-
"epoch":
|
| 9 |
"epoch_eta_seconds": 0.0,
|
| 10 |
"epochs": 45,
|
| 11 |
-
"eta_seconds":
|
| 12 |
-
"evaluated":
|
| 13 |
-
"global_step":
|
| 14 |
-
"gpu_memory_gib": 4.
|
| 15 |
-
"gpu_peak_memory_gib": 13.
|
| 16 |
-
"gradient_norm":
|
| 17 |
-
"latest_chunk": "validation/
|
| 18 |
"learning_rate": 0.0001,
|
| 19 |
-
"loss_epsilon": 0.
|
| 20 |
-
"loss_real": 0.
|
| 21 |
-
"loss_reg_real":
|
| 22 |
-
"loss_reg_synthetic":
|
| 23 |
-
"loss_synthetic":
|
| 24 |
-
"loss_total":
|
| 25 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 26 |
"operation": "official_forward_backward_optimizer",
|
| 27 |
"phase": "VALIDATING",
|
| 28 |
"pid": 10002,
|
| 29 |
"recovery_segment_id": "20261105T023238Z",
|
| 30 |
"samples": 5400,
|
| 31 |
-
"step_duration_seconds": 4.
|
| 32 |
-
"throughput": 161.
|
| 33 |
-
"timestamp_utc": "2026-11-05T02:
|
| 34 |
"total_batches": 43,
|
| 35 |
"train_run_id": "experiment_E45_spc_k0",
|
| 36 |
-
"training_throughput":
|
| 37 |
-
"updated_unix":
|
| 38 |
-
"updated_utc": "2026-11-05T02:
|
| 39 |
}
|
|
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
+
"checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
|
| 6 |
+
"completed_batches": 12,
|
| 7 |
+
"cpu_rss_gib": 6.313755035400391,
|
| 8 |
+
"epoch": 15,
|
| 9 |
"epoch_eta_seconds": 0.0,
|
| 10 |
"epochs": 45,
|
| 11 |
+
"eta_seconds": 23.86163430660963,
|
| 12 |
+
"evaluated": 1536,
|
| 13 |
+
"global_step": 45,
|
| 14 |
+
"gpu_memory_gib": 4.20784854888916,
|
| 15 |
+
"gpu_peak_memory_gib": 13.199567317962646,
|
| 16 |
+
"gradient_norm": 2.7890217304229736,
|
| 17 |
+
"latest_chunk": "validation/epoch_015/chunks/00012.npz",
|
| 18 |
"learning_rate": 0.0001,
|
| 19 |
+
"loss_epsilon": 0.13382680714130402,
|
| 20 |
+
"loss_real": 0.01956470124423504,
|
| 21 |
+
"loss_reg_real": 0.1038849875330925,
|
| 22 |
+
"loss_reg_synthetic": 0.36285677552223206,
|
| 23 |
+
"loss_synthetic": 0.15807963907718658,
|
| 24 |
+
"loss_total": 0.07597584277391434,
|
| 25 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 26 |
"operation": "official_forward_backward_optimizer",
|
| 27 |
"phase": "VALIDATING",
|
| 28 |
"pid": 10002,
|
| 29 |
"recovery_segment_id": "20261105T023238Z",
|
| 30 |
"samples": 5400,
|
| 31 |
+
"step_duration_seconds": 4.581945180892944,
|
| 32 |
+
"throughput": 161.9335855352405,
|
| 33 |
+
"timestamp_utc": "2026-11-05T02:44:08Z",
|
| 34 |
"total_batches": 43,
|
| 35 |
"train_run_id": "experiment_E45_spc_k0",
|
| 36 |
+
"training_throughput": 20.951800209292596,
|
| 37 |
+
"updated_unix": 1793846658.5428693,
|
| 38 |
+
"updated_utc": "2026-11-05T02:44:18Z"
|
| 39 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/train_steps.csv
CHANGED
|
@@ -5,3 +5,42 @@ timestamp_utc,train_run_id,recovery_segment_id,epoch,batch,global_step,loss_tota
|
|
| 5 |
2026-11-05T02:33:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,1,4,1.7940089702606201,1.3266286849975586,1.336653470993042,0.6463867425918579,2.305392026901245,2.1597659587860107,0.0001,4.6243250370025635,27.679715196440473,4.21370792388916,13.199361324310303,6.339874267578125,3.7395637035369873
|
| 6 |
2026-11-05T02:33:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,2,5,1.710416316986084,1.2274569272994995,1.4349440336227417,0.6717017292976379,2.1863515377044678,2.067314863204956,0.0001,4.548011302947998,28.144169280544894,4.21419620513916,13.199361324310303,6.3444061279296875,3.4725263118743896
|
| 7 |
2026-11-05T02:33:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,3,6,1.415545105934143,0.9928121566772461,1.3028115034103394,0.5022993087768555,1.8100521564483643,1.9769306182861328,0.0001,4.50909423828125,21.29030686140477,4.20736026763916,13.199361324310303,6.345935821533203,4.890103340148926
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
2026-11-05T02:33:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,1,4,1.7940089702606201,1.3266286849975586,1.336653470993042,0.6463867425918579,2.305392026901245,2.1597659587860107,0.0001,4.6243250370025635,27.679715196440473,4.21370792388916,13.199361324310303,6.339874267578125,3.7395637035369873
|
| 6 |
2026-11-05T02:33:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,2,5,1.710416316986084,1.2274569272994995,1.4349440336227417,0.6717017292976379,2.1863515377044678,2.067314863204956,0.0001,4.548011302947998,28.144169280544894,4.21419620513916,13.199361324310303,6.3444061279296875,3.4725263118743896
|
| 7 |
2026-11-05T02:33:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,3,6,1.415545105934143,0.9928121566772461,1.3028115034103394,0.5022993087768555,1.8100521564483643,1.9769306182861328,0.0001,4.50909423828125,21.29030686140477,4.20736026763916,13.199361324310303,6.345935821533203,4.890103340148926
|
| 8 |
+
2026-11-05T02:34:22Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,1,7,1.2671351432800293,0.8864139318466187,1.1026084423065186,0.532727837562561,1.7116870880126953,1.9229612350463867,0.0001,4.6091248989105225,27.770998358116934,4.21419620513916,13.199391841888428,6.348064422607422,3.2753567695617676
|
| 9 |
+
2026-11-05T02:34:26Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,2,8,1.371622920036316,0.9600490927696228,1.2284905910491943,0.5581454634666443,1.8062270879745483,1.8906059265136719,0.0001,4.7241575717926025,27.094777863522836,4.21370792388916,13.199391841888428,6.3471527099609375,5.345407485961914
|
| 10 |
+
2026-11-05T02:34:31Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,3,9,1.4246119260787964,1.022498607635498,1.1166820526123047,0.7060779929161072,1.7766468524932861,1.8551563024520874,0.0001,4.492733001708984,21.367840012634336,4.20784854888916,13.199391841888428,6.348278045654297,8.87842082977295
|
| 11 |
+
2026-11-05T02:35:10Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,1,10,1.1878503561019897,0.8255103826522827,1.0721659660339355,0.5491296648979187,1.4897865056991577,1.6701234579086304,0.0001,4.607059717178345,27.783447113291448,4.21370792388916,13.199391841888428,6.348358154296875,5.862489223480225
|
| 12 |
+
2026-11-05T02:35:14Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,2,11,1.0391523838043213,0.6818434000015259,1.025932788848877,0.5843984484672546,1.5178686380386353,1.6483911275863647,0.0001,4.588322877883911,27.896903379875553,4.21419620513916,13.199391841888428,6.348392486572266,3.9853928089141846
|
| 13 |
+
2026-11-05T02:35:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,3,12,0.877471387386322,0.5594723224639893,0.9067077040672302,0.5330894589424133,1.2257851362228394,1.715860366821289,0.0001,4.472378730773926,21.465087323538814,4.20736026763916,13.199391841888428,6.3484649658203125,4.246176242828369
|
| 14 |
+
2026-11-05T02:35:57Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,1,13,0.9112675786018372,0.6004937887191772,0.875447690486908,0.5366214513778687,1.2908483743667603,1.5194084644317627,0.0001,4.4592626094818115,28.70429737146031,4.21419620513916,13.199567317962646,6.348529815673828,4.1963372230529785
|
| 15 |
+
2026-11-05T02:36:02Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,2,14,0.7571407556533813,0.46975645422935486,0.8143174648284912,0.5021079182624817,1.1514276266098022,1.4126462936401367,0.0001,4.563748598098755,28.047118996283988,4.21370792388916,13.199567317962646,6.348590850830078,4.359305381774902
|
| 16 |
+
2026-11-05T02:36:06Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,3,15,0.8074727654457092,0.5067290663719177,0.8470979928970337,0.5958570837974548,1.0209497213363647,1.5450220108032227,0.0001,4.366473197937012,21.985706919111802,4.20821475982666,13.199567317962646,6.348659515380859,5.185586929321289
|
| 17 |
+
2026-11-05T02:36:45Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,1,16,0.7540974020957947,0.48116928339004517,0.7735217809677124,0.47728967666625977,1.0561646223068237,1.4124089479446411,0.0001,4.513085842132568,28.361968834059706,4.21370792388916,13.199567317962646,6.352695465087891,4.249288082122803
|
| 18 |
+
2026-11-05T02:36:49Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,2,17,0.5483358502388,0.31128787994384766,0.6802616119384766,0.40542393922805786,0.8898597955703735,1.2429455518722534,0.0001,4.580164432525635,27.94659490629185,4.21419620513916,13.199567317962646,6.352790832519531,3.7728734016418457
|
| 19 |
+
2026-11-05T02:36:53Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,3,18,0.5908375382423401,0.32776927947998047,0.8083286285400391,0.4384499490261078,0.7986207008361816,1.2806344032287598,0.0001,4.276374340057373,22.448923402414774,4.20736026763916,13.199567317962646,6.352806091308594,6.730944633483887
|
| 20 |
+
2026-11-05T02:37:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,1,19,0.5101187825202942,0.27780306339263916,0.687814474105835,0.3979944884777069,0.7724456191062927,1.2027769088745117,0.0001,4.736155033111572,27.026142325393053,4.21419620513916,13.199567317962646,6.352817535400391,3.4259095191955566
|
| 21 |
+
2026-11-05T02:37:36Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,2,20,0.37807542085647583,0.20820730924606323,0.46521008014678955,0.280882865190506,0.7042987942695618,1.0282939672470093,0.0001,4.497614622116089,28.459530385415082,4.21370792388916,13.199567317962646,6.352817535400391,3.5623104572296143
|
| 22 |
+
2026-11-05T02:37:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,3,21,0.5498836040496826,0.3616764545440674,0.5351462960243225,0.3802229166030884,0.5518221259117126,1.0541349649429321,0.0001,4.428439140319824,21.678066912096444,4.20784854888916,13.199567317962646,6.35284423828125,5.640958309173584
|
| 23 |
+
2026-11-05T02:38:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,1,22,0.37231922149658203,0.19556286931037903,0.4724317193031311,0.38374873995780945,0.5717012286186218,1.051354169845581,0.0001,4.457226276397705,28.717411246944497,4.21370792388916,13.199567317962646,6.352836608886719,2.94069766998291
|
| 24 |
+
2026-11-05T02:38:24Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,2,23,0.30827227234840393,0.144283726811409,0.4486972689628601,0.35716506838798523,0.4893544316291809,0.9479215741157532,0.0001,4.544239521026611,28.167529331966836,4.21419620513916,13.199567317962646,6.352851867675781,3.106137752532959
|
| 25 |
+
2026-11-05T02:38:28Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,3,24,0.3707720637321472,0.1848861277103424,0.5284978151321411,0.40896984934806824,0.44575411081314087,1.0729613304138184,0.0001,4.430412530899048,21.668411085980534,4.20736026763916,13.199567317962646,6.352886199951172,4.545416355133057
|
| 26 |
+
2026-11-05T02:39:06Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,1,25,0.23698531091213226,0.10441966354846954,0.3608815670013428,0.28275266289711,0.378287136554718,0.8491285443305969,0.0001,4.535761594772339,28.220178094793503,4.21419620513916,13.199567317962646,6.3529052734375,2.5497570037841797
|
| 27 |
+
2026-11-05T02:39:11Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,2,26,0.27532851696014404,0.13280969858169556,0.398764044046402,0.3139715790748596,0.3783737123012543,0.8116950392723083,0.0001,4.540703773498535,28.189462776026495,4.21370792388916,13.199567317962646,6.352931976318359,6.295091152191162
|
| 28 |
+
2026-11-05T02:39:15Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,3,27,0.22572989761829376,0.10692095756530762,0.3213772773742676,0.267505943775177,0.3405652940273285,0.7080139517784119,0.0001,4.302645921707153,22.311852229269714,4.20821475982666,13.199567317962646,6.352973937988281,5.602907180786133
|
| 29 |
+
2026-11-05T02:39:53Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,1,28,0.18336300551891327,0.07133440673351288,0.29729902744293213,0.26639851927757263,0.26976266503334045,0.7569214105606079,0.0001,4.613707780838013,27.74341290786099,4.21370792388916,13.199567317962646,6.352973937988281,2.6516027450561523
|
| 30 |
+
2026-11-05T02:39:58Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,2,29,0.2577931582927704,0.10022659599781036,0.44544264674186707,0.38612720370292664,0.32152873277664185,0.8502084612846375,0.0001,4.586455821990967,27.90825966016513,4.21419620513916,13.199567317962646,6.352993011474609,4.791995525360107
|
| 31 |
+
2026-11-05T02:40:02Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,3,30,0.1792241930961609,0.05060635507106781,0.36566129326820374,0.3099075257778168,0.23522202670574188,0.75429767370224,0.0001,4.429757595062256,21.671614741856054,4.20736026763916,13.199567317962646,6.353000640869141,3.241050958633423
|
| 32 |
+
2026-11-05T02:40:42Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,1,31,0.1871512085199356,0.06394229829311371,0.3720165193080902,0.2581752836704254,0.23884811997413635,0.6717081069946289,0.0001,4.6319260597229,27.634292592239145,4.21419620513916,13.199567317962646,6.313625335693359,4.2985382080078125
|
| 33 |
+
2026-11-05T02:40:47Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,2,32,0.13410496711730957,0.05497533455491066,0.2134230136871338,0.16946427524089813,0.21618342399597168,0.5425635576248169,0.0001,4.8092944622039795,26.615130557287774,4.21370792388916,13.199567317962646,6.313652038574219,2.4699645042419434
|
| 34 |
+
2026-11-05T02:40:52Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,3,33,0.14285658299922943,0.045638084411621094,0.28416675329208374,0.19439589977264404,0.1863006353378296,0.6746764183044434,0.0001,4.476435422897339,21.445634959671715,4.20784854888916,13.199567317962646,6.31365966796875,2.234697103500366
|
| 35 |
+
2026-11-05T02:41:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,1,34,0.15002411603927612,0.041587233543395996,0.2889729142189026,0.2995773255825043,0.1665458232164383,0.7011368870735168,0.0001,4.786452054977417,26.742146067647997,4.21370792388916,13.199567317962646,6.313663482666016,4.473418235778809
|
| 36 |
+
2026-11-05T02:41:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,2,35,0.09535311907529831,0.03432588279247284,0.16249723732471466,0.1297117918729782,0.16799309849739075,0.44184446334838867,0.0001,4.679177761077881,27.35523344821897,4.21419620513916,13.199567317962646,6.31365966796875,1.8804813623428345
|
| 37 |
+
2026-11-05T02:41:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,3,36,0.11845378577709198,0.041674211621284485,0.21308833360671997,0.1828250139951706,0.14130644500255585,0.5113574862480164,0.0001,4.5488574504852295,21.10420057013649,4.20736026763916,13.199567317962646,6.3136749267578125,3.603003978729248
|
| 38 |
+
2026-11-05T02:42:21Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,1,37,0.08822737634181976,0.03273775428533554,0.1622830331325531,0.10143797844648361,0.13649021089076996,0.3714805841445923,0.0001,4.575104475021362,27.97750318027488,4.21419620513916,13.199567317962646,6.313690185546875,1.5368391275405884
|
| 39 |
+
2026-11-05T02:42:26Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,2,38,0.10865352302789688,0.0280852522701025,0.21965403854846954,0.20935462415218353,0.12176745384931564,0.5415651798248291,0.0001,4.667818307876587,27.421804268604408,4.21370792388916,13.199567317962646,6.313724517822266,2.45637583732605
|
| 40 |
+
2026-11-05T02:42:30Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,3,39,0.07437027245759964,0.02421887405216694,0.13256892561912537,0.11240734905004501,0.1274520456790924,0.36493992805480957,0.0001,4.467528581619263,21.488390783882718,4.20821475982666,13.199567317962646,6.313724517822266,1.1801849603652954
|
| 41 |
+
2026-11-05T02:43:10Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,1,40,0.11135122925043106,0.017928674817085266,0.24642586708068848,0.2756049335002899,0.10229283571243286,0.6242587566375732,0.0001,4.674060106277466,27.38518484777088,4.21370792388916,13.199567317962646,6.313732147216797,3.9312450885772705
|
| 42 |
+
2026-11-05T02:43:15Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,2,41,0.07282369583845139,0.020908314734697342,0.14373445510864258,0.11426927894353867,0.10161476582288742,0.3838481903076172,0.0001,4.7962212562561035,26.687676226996228,4.21419620513916,13.199567317962646,6.313728332519531,2.085524082183838
|
| 43 |
+
2026-11-05T02:43:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,3,42,0.06252893805503845,0.02631022036075592,0.09998758137226105,0.06848233938217163,0.09765395522117615,0.27334070205688477,0.0001,4.421590089797974,21.71164627438052,4.20736026763916,13.199567317962646,6.313732147216797,1.676707148551941
|
| 44 |
+
2026-11-05T02:43:59Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,1,43,0.08387480676174164,0.01848646253347397,0.19105781614780426,0.16103018820285797,0.08879290521144867,0.37610241770744324,0.0001,4.769833326339722,26.83531923288916,4.21419620513916,13.199567317962646,6.313716888427734,4.288538455963135
|
| 45 |
+
2026-11-05T02:44:04Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,2,44,0.07876357436180115,0.01577068120241165,0.1664869785308838,0.18699847161769867,0.08303209394216537,0.38371822237968445,0.0001,4.5843305587768555,27.92119773189996,4.21370792388916,13.199567317962646,6.313720703125,3.8296756744384766
|
| 46 |
+
2026-11-05T02:44:08Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,3,45,0.07597584277391434,0.01956470124423504,0.15807963907718658,0.13382680714130402,0.1038849875330925,0.36285677552223206,0.0001,4.581945180892944,20.951800209292596,4.20784854888916,13.199567317962646,6.313755035400391,2.7890217304229736
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/live_telemetry.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
{
|
| 2 |
-
"gpu_utilization":
|
| 3 |
"gpu_memory_mib": 15867.0,
|
| 4 |
-
"timestamp_unix":
|
| 5 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"gpu_utilization": 41.0,
|
| 3 |
"gpu_memory_mib": 15867.0,
|
| 4 |
+
"timestamp_unix": 1793846655.7590516
|
| 5 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/worker_status.json
CHANGED
|
@@ -2,38 +2,38 @@
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
-
"checkpoint_identity": "
|
| 6 |
-
"completed_batches":
|
| 7 |
-
"cpu_rss_gib": 6.
|
| 8 |
-
"epoch":
|
| 9 |
"epoch_eta_seconds": 0.0,
|
| 10 |
"epochs": 45,
|
| 11 |
-
"eta_seconds":
|
| 12 |
-
"evaluated":
|
| 13 |
-
"global_step":
|
| 14 |
-
"gpu_memory_gib": 4.
|
| 15 |
-
"gpu_peak_memory_gib": 13.
|
| 16 |
-
"gradient_norm":
|
| 17 |
-
"latest_chunk": "validation/
|
| 18 |
"learning_rate": 0.0001,
|
| 19 |
-
"loss_epsilon": 0.
|
| 20 |
-
"loss_real": 0.
|
| 21 |
-
"loss_reg_real":
|
| 22 |
-
"loss_reg_synthetic":
|
| 23 |
-
"loss_synthetic":
|
| 24 |
-
"loss_total":
|
| 25 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 26 |
"operation": "official_forward_backward_optimizer",
|
| 27 |
"phase": "VALIDATING",
|
| 28 |
"pid": 10002,
|
| 29 |
"recovery_segment_id": "20261105T023238Z",
|
| 30 |
"samples": 5400,
|
| 31 |
-
"step_duration_seconds": 4.
|
| 32 |
-
"throughput": 161.
|
| 33 |
-
"timestamp_utc": "2026-11-05T02:
|
| 34 |
"total_batches": 43,
|
| 35 |
"train_run_id": "experiment_E45_spc_k0",
|
| 36 |
-
"training_throughput":
|
| 37 |
-
"updated_unix":
|
| 38 |
-
"updated_utc": "2026-11-05T02:
|
| 39 |
}
|
|
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
+
"checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
|
| 6 |
+
"completed_batches": 12,
|
| 7 |
+
"cpu_rss_gib": 6.313755035400391,
|
| 8 |
+
"epoch": 15,
|
| 9 |
"epoch_eta_seconds": 0.0,
|
| 10 |
"epochs": 45,
|
| 11 |
+
"eta_seconds": 23.86163430660963,
|
| 12 |
+
"evaluated": 1536,
|
| 13 |
+
"global_step": 45,
|
| 14 |
+
"gpu_memory_gib": 4.20784854888916,
|
| 15 |
+
"gpu_peak_memory_gib": 13.199567317962646,
|
| 16 |
+
"gradient_norm": 2.7890217304229736,
|
| 17 |
+
"latest_chunk": "validation/epoch_015/chunks/00012.npz",
|
| 18 |
"learning_rate": 0.0001,
|
| 19 |
+
"loss_epsilon": 0.13382680714130402,
|
| 20 |
+
"loss_real": 0.01956470124423504,
|
| 21 |
+
"loss_reg_real": 0.1038849875330925,
|
| 22 |
+
"loss_reg_synthetic": 0.36285677552223206,
|
| 23 |
+
"loss_synthetic": 0.15807963907718658,
|
| 24 |
+
"loss_total": 0.07597584277391434,
|
| 25 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 26 |
"operation": "official_forward_backward_optimizer",
|
| 27 |
"phase": "VALIDATING",
|
| 28 |
"pid": 10002,
|
| 29 |
"recovery_segment_id": "20261105T023238Z",
|
| 30 |
"samples": 5400,
|
| 31 |
+
"step_duration_seconds": 4.581945180892944,
|
| 32 |
+
"throughput": 161.9335855352405,
|
| 33 |
+
"timestamp_utc": "2026-11-05T02:44:08Z",
|
| 34 |
"total_batches": 43,
|
| 35 |
"train_run_id": "experiment_E45_spc_k0",
|
| 36 |
+
"training_throughput": 20.951800209292596,
|
| 37 |
+
"updated_unix": 1793846658.5428693,
|
| 38 |
+
"updated_utc": "2026-11-05T02:44:18Z"
|
| 39 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/state.json
CHANGED
|
@@ -2,69 +2,69 @@
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
-
"best_acc": 0.
|
| 6 |
-
"best_epoch":
|
| 7 |
-
"checkpoint_identity": "
|
| 8 |
-
"completed_batches":
|
| 9 |
-
"cpu_rss_gib": 6.
|
| 10 |
-
"epoch":
|
| 11 |
"epoch_eta_seconds": 0.0,
|
| 12 |
"epochs": 45,
|
| 13 |
-
"eta_seconds":
|
| 14 |
-
"evaluated":
|
| 15 |
"evaluation_progress": {
|
| 16 |
"arm": "spc_k0",
|
| 17 |
"batch": 3,
|
| 18 |
"batches": 3,
|
| 19 |
-
"checkpoint_identity": "
|
| 20 |
-
"completed_batches":
|
| 21 |
-
"cpu_rss_gib": 6.
|
| 22 |
-
"epoch":
|
| 23 |
"epoch_eta_seconds": 0.0,
|
| 24 |
"epochs": 45,
|
| 25 |
-
"eta_seconds":
|
| 26 |
-
"evaluated":
|
| 27 |
-
"global_step":
|
| 28 |
-
"gpu_memory_gib": 4.
|
| 29 |
-
"gpu_peak_memory_gib": 13.
|
| 30 |
-
"gradient_norm":
|
| 31 |
-
"latest_chunk": "validation/
|
| 32 |
"learning_rate": 0.0001,
|
| 33 |
-
"loss_epsilon": 0.
|
| 34 |
-
"loss_real": 0.
|
| 35 |
-
"loss_reg_real":
|
| 36 |
-
"loss_reg_synthetic":
|
| 37 |
-
"loss_synthetic":
|
| 38 |
-
"loss_total":
|
| 39 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 40 |
"operation": "official_forward_backward_optimizer",
|
| 41 |
"phase": "VALIDATING",
|
| 42 |
"pid": 10002,
|
| 43 |
"recovery_segment_id": "20261105T023238Z",
|
| 44 |
"samples": 5400,
|
| 45 |
-
"step_duration_seconds": 4.
|
| 46 |
-
"throughput": 162.
|
| 47 |
-
"timestamp_utc": "2026-11-05T02:
|
| 48 |
"total_batches": 43,
|
| 49 |
"train_run_id": "experiment_E45_spc_k0",
|
| 50 |
-
"training_throughput":
|
| 51 |
-
"updated_unix":
|
| 52 |
-
"updated_utc": "2026-11-05T02:
|
| 53 |
},
|
| 54 |
"experiment_id": "SPC_EUROSAT_HAIHT_PROMPTS_45E",
|
| 55 |
-
"global_step":
|
| 56 |
-
"gpu_memory_gib": 4.
|
| 57 |
-
"gpu_peak_memory_gib": 13.
|
| 58 |
-
"gradient_norm":
|
| 59 |
-
"heartbeat": "2026-11-05T02:
|
| 60 |
-
"latest_chunk": "validation/
|
| 61 |
"learning_rate": 0.0001,
|
| 62 |
-
"loss_epsilon": 0.
|
| 63 |
-
"loss_real": 0.
|
| 64 |
-
"loss_reg_real":
|
| 65 |
-
"loss_reg_synthetic":
|
| 66 |
-
"loss_synthetic":
|
| 67 |
-
"loss_total":
|
| 68 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 69 |
"operation": "official_forward_backward_optimizer",
|
| 70 |
"phase": "VALIDATING",
|
|
@@ -72,14 +72,14 @@
|
|
| 72 |
"recovery_segment_id": "20261105T023238Z",
|
| 73 |
"samples": 5400,
|
| 74 |
"stage": "VALIDATING",
|
| 75 |
-
"step_duration_seconds": 4.
|
| 76 |
-
"throughput": 162.
|
| 77 |
-
"timestamp_utc": "2026-11-05T02:
|
| 78 |
"total_batches": 43,
|
| 79 |
"total_epochs": 45,
|
| 80 |
"train_run_id": "d45_e45_20261104T122347Z_E45",
|
| 81 |
-
"training_throughput":
|
| 82 |
-
"updated_unix":
|
| 83 |
-
"updated_utc": "2026-11-05T02:
|
| 84 |
"worker_pid": 10002
|
| 85 |
}
|
|
|
|
| 2 |
"arm": "spc_k0",
|
| 3 |
"batch": 3,
|
| 4 |
"batches": 3,
|
| 5 |
+
"best_acc": 0.9007407407407407,
|
| 6 |
+
"best_epoch": 14,
|
| 7 |
+
"checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
|
| 8 |
+
"completed_batches": 9,
|
| 9 |
+
"cpu_rss_gib": 6.313755035400391,
|
| 10 |
+
"epoch": 15,
|
| 11 |
"epoch_eta_seconds": 0.0,
|
| 12 |
"epochs": 45,
|
| 13 |
+
"eta_seconds": 26.221441715955734,
|
| 14 |
+
"evaluated": 1152,
|
| 15 |
"evaluation_progress": {
|
| 16 |
"arm": "spc_k0",
|
| 17 |
"batch": 3,
|
| 18 |
"batches": 3,
|
| 19 |
+
"checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
|
| 20 |
+
"completed_batches": 9,
|
| 21 |
+
"cpu_rss_gib": 6.313755035400391,
|
| 22 |
+
"epoch": 15,
|
| 23 |
"epoch_eta_seconds": 0.0,
|
| 24 |
"epochs": 45,
|
| 25 |
+
"eta_seconds": 26.221441715955734,
|
| 26 |
+
"evaluated": 1152,
|
| 27 |
+
"global_step": 45,
|
| 28 |
+
"gpu_memory_gib": 4.20784854888916,
|
| 29 |
+
"gpu_peak_memory_gib": 13.199567317962646,
|
| 30 |
+
"gradient_norm": 2.7890217304229736,
|
| 31 |
+
"latest_chunk": "validation/epoch_015/chunks/00009.npz",
|
| 32 |
"learning_rate": 0.0001,
|
| 33 |
+
"loss_epsilon": 0.13382680714130402,
|
| 34 |
+
"loss_real": 0.01956470124423504,
|
| 35 |
+
"loss_reg_real": 0.1038849875330925,
|
| 36 |
+
"loss_reg_synthetic": 0.36285677552223206,
|
| 37 |
+
"loss_synthetic": 0.15807963907718658,
|
| 38 |
+
"loss_total": 0.07597584277391434,
|
| 39 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 40 |
"operation": "official_forward_backward_optimizer",
|
| 41 |
"phase": "VALIDATING",
|
| 42 |
"pid": 10002,
|
| 43 |
"recovery_segment_id": "20261105T023238Z",
|
| 44 |
"samples": 5400,
|
| 45 |
+
"step_duration_seconds": 4.581945180892944,
|
| 46 |
+
"throughput": 162.0048220847862,
|
| 47 |
+
"timestamp_utc": "2026-11-05T02:44:08Z",
|
| 48 |
"total_batches": 43,
|
| 49 |
"train_run_id": "experiment_E45_spc_k0",
|
| 50 |
+
"training_throughput": 20.951800209292596,
|
| 51 |
+
"updated_unix": 1793846656.168427,
|
| 52 |
+
"updated_utc": "2026-11-05T02:44:16Z"
|
| 53 |
},
|
| 54 |
"experiment_id": "SPC_EUROSAT_HAIHT_PROMPTS_45E",
|
| 55 |
+
"global_step": 45,
|
| 56 |
+
"gpu_memory_gib": 4.20784854888916,
|
| 57 |
+
"gpu_peak_memory_gib": 13.199567317962646,
|
| 58 |
+
"gradient_norm": 2.7890217304229736,
|
| 59 |
+
"heartbeat": "2026-11-05T02:44:16Z",
|
| 60 |
+
"latest_chunk": "validation/epoch_015/chunks/00009.npz",
|
| 61 |
"learning_rate": 0.0001,
|
| 62 |
+
"loss_epsilon": 0.13382680714130402,
|
| 63 |
+
"loss_real": 0.01956470124423504,
|
| 64 |
+
"loss_reg_real": 0.1038849875330925,
|
| 65 |
+
"loss_reg_synthetic": 0.36285677552223206,
|
| 66 |
+
"loss_synthetic": 0.15807963907718658,
|
| 67 |
+
"loss_total": 0.07597584277391434,
|
| 68 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 69 |
"operation": "official_forward_backward_optimizer",
|
| 70 |
"phase": "VALIDATING",
|
|
|
|
| 72 |
"recovery_segment_id": "20261105T023238Z",
|
| 73 |
"samples": 5400,
|
| 74 |
"stage": "VALIDATING",
|
| 75 |
+
"step_duration_seconds": 4.581945180892944,
|
| 76 |
+
"throughput": 162.0048220847862,
|
| 77 |
+
"timestamp_utc": "2026-11-05T02:44:08Z",
|
| 78 |
"total_batches": 43,
|
| 79 |
"total_epochs": 45,
|
| 80 |
"train_run_id": "d45_e45_20261104T122347Z_E45",
|
| 81 |
+
"training_throughput": 20.951800209292596,
|
| 82 |
+
"updated_unix": 1793846656.168427,
|
| 83 |
+
"updated_utc": "2026-11-05T02:44:16Z",
|
| 84 |
"worker_pid": 10002
|
| 85 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00041.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5d7868fa5a6fc0d2266966f6e96db483e35b02b8d7c44d04f72bfa210332e6df
|
| 3 |
+
size 7747
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00042.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:77142e529728ee7fa716220ce39fe0856bd2b63bffd7559078510d483569e218
|
| 3 |
+
size 7744
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00043.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5cbea9f035b0cbbb43533318bea3bbe389b5010aa44e2c03973cf69700bb68d
|
| 3 |
+
size 2732
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/metrics.json
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"accuracy": 0.6287037037037037,
|
| 3 |
+
"checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
|
| 4 |
+
"confusion_matrix": [
|
| 5 |
+
[
|
| 6 |
+
477,
|
| 7 |
+
2,
|
| 8 |
+
30,
|
| 9 |
+
19,
|
| 10 |
+
5,
|
| 11 |
+
37,
|
| 12 |
+
16,
|
| 13 |
+
2,
|
| 14 |
+
9,
|
| 15 |
+
3
|
| 16 |
+
],
|
| 17 |
+
[
|
| 18 |
+
1,
|
| 19 |
+
544,
|
| 20 |
+
23,
|
| 21 |
+
1,
|
| 22 |
+
0,
|
| 23 |
+
3,
|
| 24 |
+
0,
|
| 25 |
+
1,
|
| 26 |
+
6,
|
| 27 |
+
21
|
| 28 |
+
],
|
| 29 |
+
[
|
| 30 |
+
8,
|
| 31 |
+
226,
|
| 32 |
+
19,
|
| 33 |
+
51,
|
| 34 |
+
3,
|
| 35 |
+
168,
|
| 36 |
+
2,
|
| 37 |
+
15,
|
| 38 |
+
103,
|
| 39 |
+
5
|
| 40 |
+
],
|
| 41 |
+
[
|
| 42 |
+
16,
|
| 43 |
+
0,
|
| 44 |
+
0,
|
| 45 |
+
286,
|
| 46 |
+
84,
|
| 47 |
+
30,
|
| 48 |
+
2,
|
| 49 |
+
34,
|
| 50 |
+
48,
|
| 51 |
+
0
|
| 52 |
+
],
|
| 53 |
+
[
|
| 54 |
+
0,
|
| 55 |
+
0,
|
| 56 |
+
0,
|
| 57 |
+
1,
|
| 58 |
+
333,
|
| 59 |
+
1,
|
| 60 |
+
3,
|
| 61 |
+
162,
|
| 62 |
+
0,
|
| 63 |
+
0
|
| 64 |
+
],
|
| 65 |
+
[
|
| 66 |
+
118,
|
| 67 |
+
25,
|
| 68 |
+
0,
|
| 69 |
+
1,
|
| 70 |
+
0,
|
| 71 |
+
250,
|
| 72 |
+
0,
|
| 73 |
+
2,
|
| 74 |
+
4,
|
| 75 |
+
0
|
| 76 |
+
],
|
| 77 |
+
[
|
| 78 |
+
307,
|
| 79 |
+
13,
|
| 80 |
+
6,
|
| 81 |
+
8,
|
| 82 |
+
29,
|
| 83 |
+
66,
|
| 84 |
+
30,
|
| 85 |
+
32,
|
| 86 |
+
9,
|
| 87 |
+
0
|
| 88 |
+
],
|
| 89 |
+
[
|
| 90 |
+
0,
|
| 91 |
+
1,
|
| 92 |
+
1,
|
| 93 |
+
0,
|
| 94 |
+
4,
|
| 95 |
+
1,
|
| 96 |
+
3,
|
| 97 |
+
590,
|
| 98 |
+
0,
|
| 99 |
+
0
|
| 100 |
+
],
|
| 101 |
+
[
|
| 102 |
+
22,
|
| 103 |
+
3,
|
| 104 |
+
1,
|
| 105 |
+
27,
|
| 106 |
+
17,
|
| 107 |
+
13,
|
| 108 |
+
0,
|
| 109 |
+
4,
|
| 110 |
+
406,
|
| 111 |
+
7
|
| 112 |
+
],
|
| 113 |
+
[
|
| 114 |
+
6,
|
| 115 |
+
10,
|
| 116 |
+
107,
|
| 117 |
+
0,
|
| 118 |
+
3,
|
| 119 |
+
4,
|
| 120 |
+
0,
|
| 121 |
+
0,
|
| 122 |
+
10,
|
| 123 |
+
460
|
| 124 |
+
]
|
| 125 |
+
],
|
| 126 |
+
"duration_seconds": 33.647279500961304,
|
| 127 |
+
"evaluation_identity": "13c57cd97dfa4bbb8efaecefc576a7c655e496fd8983f8d5f6f7efd3fbdc272d",
|
| 128 |
+
"image_manifest_hash": "d00f5f6e1e711aada1d545d6d1aee03cdc9d5a9e0c6431ec4ce6551002a3d17f",
|
| 129 |
+
"macro_f1": 0.5767701775459999,
|
| 130 |
+
"macro_precision": 0.5966315282484262,
|
| 131 |
+
"macro_recall": 0.6218333333333333,
|
| 132 |
+
"peak_gpu_memory_gib": 13.199361324310303,
|
| 133 |
+
"per_class": {
|
| 134 |
+
"accuracy": [
|
| 135 |
+
0.795,
|
| 136 |
+
0.9066666666666666,
|
| 137 |
+
0.03166666666666667,
|
| 138 |
+
0.572,
|
| 139 |
+
0.666,
|
| 140 |
+
0.625,
|
| 141 |
+
0.06,
|
| 142 |
+
0.9833333333333333,
|
| 143 |
+
0.812,
|
| 144 |
+
0.7666666666666667
|
| 145 |
+
],
|
| 146 |
+
"f1": [
|
| 147 |
+
0.6135048231511254,
|
| 148 |
+
0.7640449438202247,
|
| 149 |
+
0.048284625158831,
|
| 150 |
+
0.639821029082774,
|
| 151 |
+
0.6809815950920245,
|
| 152 |
+
0.513874614594039,
|
| 153 |
+
0.1079136690647482,
|
| 154 |
+
0.8183079056865464,
|
| 155 |
+
0.7415525114155251,
|
| 156 |
+
0.8394160583941606
|
| 157 |
+
],
|
| 158 |
+
"precision": [
|
| 159 |
+
0.49947643979057593,
|
| 160 |
+
0.6601941747572816,
|
| 161 |
+
0.10160427807486631,
|
| 162 |
+
0.7258883248730964,
|
| 163 |
+
0.696652719665272,
|
| 164 |
+
0.4363001745200698,
|
| 165 |
+
0.5357142857142857,
|
| 166 |
+
0.7007125890736342,
|
| 167 |
+
0.6823529411764706,
|
| 168 |
+
0.9274193548387096
|
| 169 |
+
],
|
| 170 |
+
"recall": [
|
| 171 |
+
0.795,
|
| 172 |
+
0.9066666666666666,
|
| 173 |
+
0.03166666666666667,
|
| 174 |
+
0.572,
|
| 175 |
+
0.666,
|
| 176 |
+
0.625,
|
| 177 |
+
0.06,
|
| 178 |
+
0.9833333333333333,
|
| 179 |
+
0.812,
|
| 180 |
+
0.7666666666666667
|
| 181 |
+
],
|
| 182 |
+
"support": [
|
| 183 |
+
600,
|
| 184 |
+
600,
|
| 185 |
+
600,
|
| 186 |
+
500,
|
| 187 |
+
500,
|
| 188 |
+
400,
|
| 189 |
+
500,
|
| 190 |
+
600,
|
| 191 |
+
500,
|
| 192 |
+
600
|
| 193 |
+
]
|
| 194 |
+
},
|
| 195 |
+
"sample_count": 5400,
|
| 196 |
+
"timestamp_utc": "2026-11-05T02:34:15Z"
|
| 197 |
+
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/predictions.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/progress.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
| 1 |
{
|
| 2 |
"checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
|
| 3 |
-
"completed_batches":
|
| 4 |
-
"eta_seconds":
|
| 5 |
-
"evaluated":
|
| 6 |
-
"latest_chunk": "validation/epoch_002/chunks/
|
| 7 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 8 |
"samples": 5400,
|
| 9 |
-
"throughput":
|
| 10 |
"total_batches": 43,
|
| 11 |
-
"updated_utc": "2026-11-05T02:34:
|
| 12 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
|
| 3 |
+
"completed_batches": 43,
|
| 4 |
+
"eta_seconds": 0.0,
|
| 5 |
+
"evaluated": 5400,
|
| 6 |
+
"latest_chunk": "validation/epoch_002/chunks/00043.npz",
|
| 7 |
"metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
|
| 8 |
"samples": 5400,
|
| 9 |
+
"throughput": 160.72345090721953,
|
| 10 |
"total_batches": 43,
|
| 11 |
+
"updated_utc": "2026-11-05T02:34:15Z"
|
| 12 |
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/recovery_journal.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"duration_basis": "measured optimizer steps + evaluator; excludes checkpoint serialization",
|
| 3 |
+
"epoch_duration_seconds": 47.328710079193115,
|
| 4 |
+
"validation_seconds": 35.53907918930054
|
| 5 |
+
}
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00001.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0b4effea2f1682bfcadc86f2d36d1eb998f692d5348adc4fa545b41d34a74ad8
|
| 3 |
+
size 7732
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00002.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bdb765d40472253d1a4d8e6a57af47b18ae41655672582d87712355a33ba2146
|
| 3 |
+
size 7746
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00003.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c5191510ec3d246bf6bed5ed1ec37dbc12f7be0abee642db8a27b328354ae9f6
|
| 3 |
+
size 7748
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00004.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:af65422afaa074470034cc706bca690b53a284552784f5b8583ba3f434355ce1
|
| 3 |
+
size 7746
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00005.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3c0c1d9d734ebd66c42831420bfc8e6941cfd113169cc5df00a30f68b3c6c8d
|
| 3 |
+
size 7721
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00006.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a47da9363fa2acd9e154a9ec85265e770f6c2ff32130b2d4c112150edf74dca0
|
| 3 |
+
size 7744
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00007.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c6ff65fb517fd248d5d177f06a5f591bd1b66b44eb6d7641d811b6b8565f323
|
| 3 |
+
size 7725
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00008.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e3414030f95ec67254688c5fc69ddeca9f2d62f589a410d7b2d0fca34f5e1ff9
|
| 3 |
+
size 7754
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00009.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc9c858709c8a36acb5c0ca74511e89ab6658c0b7e7ace327d90c7dbd2b42389
|
| 3 |
+
size 7742
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00010.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b1c4ba9ab1bb76aecee16f0a75bdb432b9bb08a533fc76ebb4a45d0af99ad268
|
| 3 |
+
size 7766
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00011.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2519ef1086f94c694dc84b5ef57ceb7ac71bf70930b7f6e032d414543f0144e5
|
| 3 |
+
size 7740
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00012.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e45aa94f4b1f11e241c27f51d3b6bd183541cb2c0917f0c92947afa05292c028
|
| 3 |
+
size 7763
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00013.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5117bf312ab6eb8aaeb07fa1a87dd1ae198b1de1775f57a9387c25c8edafb275
|
| 3 |
+
size 7741
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00014.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:10a63aa5e8c111d7362222bfd3b4496100921a1963a7b194910d9bc8cc63736a
|
| 3 |
+
size 7753
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00015.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:563f0ea80418abb5c3d85e03b1ead50f131dfeb50975f51f3f4b488a40e014a7
|
| 3 |
+
size 7719
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00016.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4383e836a5e69637eaadb584a06a65f8aed9ad19f86314c0633d6bb83e430376
|
| 3 |
+
size 7718
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00017.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b754f471b0f041bfc751a086fbeec4c093e4fe6c28d8f315d409750d30c2d8f2
|
| 3 |
+
size 7785
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00018.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e6d52c2ef2168bad63918597fb1bca3edeed0484307e4f40bff4de61223c73fe
|
| 3 |
+
size 7734
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00019.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:fc8ded5983018c26d3b8c0d25050c52faba1014384bf129fabf2a0b5f0576794
|
| 3 |
+
size 7745
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00020.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3ac3d30ee7a564e036e4b4b82afc7275cd253550d542504e3a76836750d31f06
|
| 3 |
+
size 7762
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00021.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7505bf92cf9f94775e3f83b92c7462141e0180fa576bec611b64ce6b4d20f55b
|
| 3 |
+
size 7759
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00022.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2fcedbf386537dd8c9b77f4771e063ecd81fd4c8f0d57a81d414aeffc028e5ce
|
| 3 |
+
size 7740
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00023.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:49829107b6e97edd38f5bac1b2677a0905beae2d507230fb3ccee5ccc03e0a46
|
| 3 |
+
size 7772
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00024.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:93a110e4d2df522b0d8abdde6c75835e23d88eae88f448093f7841be829dd619
|
| 3 |
+
size 7743
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00025.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5419499a9f4d55dfbafd30a568b5ad54cb8dbeb8814ad556007d7ba76854b423
|
| 3 |
+
size 7759
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00026.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:89c371890dae84a0024a9617067dcfcb8a54b5c499596fcd30398564da61b01d
|
| 3 |
+
size 7757
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00027.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b11be05110251b5be4ec49ad8246052e535c18ed5d7481755a158be67ec2f8b3
|
| 3 |
+
size 7737
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00028.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:cc3ed5170b8a0c1ce89af7a05de530aec207352c95bb11e55da4dc3c62150e2c
|
| 3 |
+
size 7748
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00029.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:54d1eca28c69c4226e7d2f4dd000ec2958e0ec5f3304cba71e95cd4575aced88
|
| 3 |
+
size 7742
|
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00030.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:77063c60809cfb096e9d3ca12776ee58ccfb16a707ab910d6a1d05aca3172c4c
|
| 3 |
+
size 7749
|