DanielNguyen121 commited on
Commit
f3fb253
·
verified ·
1 Parent(s): 5f6f8db

Verified D45/E45 source, measured progress and independent recovery checkpoints

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/BEST.json +5 -5
  2. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/CURRENT.json +5 -5
  3. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/PREVIOUS.json +6 -6
  4. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt +3 -0
  5. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt +3 -0
  6. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/finetuning/progress.json +7 -7
  7. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/epoch_metrics.csv +13 -0
  8. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/events.jsonl +65 -0
  9. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/status.json +23 -23
  10. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/train_steps.csv +39 -0
  11. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/live_telemetry.json +2 -2
  12. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/worker_status.json +23 -23
  13. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/state.json +49 -49
  14. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00041.npz +3 -0
  15. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00042.npz +3 -0
  16. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00043.npz +3 -0
  17. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/metrics.json +197 -0
  18. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/predictions.jsonl +0 -0
  19. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/progress.json +6 -6
  20. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/recovery_journal.json +5 -0
  21. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00001.npz +3 -0
  22. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00002.npz +3 -0
  23. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00003.npz +3 -0
  24. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00004.npz +3 -0
  25. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00005.npz +3 -0
  26. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00006.npz +3 -0
  27. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00007.npz +3 -0
  28. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00008.npz +3 -0
  29. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00009.npz +3 -0
  30. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00010.npz +3 -0
  31. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00011.npz +3 -0
  32. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00012.npz +3 -0
  33. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00013.npz +3 -0
  34. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00014.npz +3 -0
  35. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00015.npz +3 -0
  36. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00016.npz +3 -0
  37. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00017.npz +3 -0
  38. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00018.npz +3 -0
  39. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00019.npz +3 -0
  40. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00020.npz +3 -0
  41. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00021.npz +3 -0
  42. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00022.npz +3 -0
  43. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00023.npz +3 -0
  44. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00024.npz +3 -0
  45. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00025.npz +3 -0
  46. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00026.npz +3 -0
  47. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00027.npz +3 -0
  48. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00028.npz +3 -0
  49. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00029.npz +3 -0
  50. experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00030.npz +3 -0
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/BEST.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "batch_next": 0,
3
  "bytes": 67597675,
4
- "epoch_index": 1,
5
- "global_step": 3,
6
- "path": "checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt",
7
  "remote_verified": false,
8
- "sha256": "1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1",
9
- "timestamp_utc": "2026-11-05T02:33:27Z"
10
  }
 
1
  {
2
  "batch_next": 0,
3
  "bytes": 67597675,
4
+ "epoch_index": 14,
5
+ "global_step": 42,
6
+ "path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt",
7
  "remote_verified": false,
8
+ "sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92",
9
+ "timestamp_utc": "2026-11-05T02:43:54Z"
10
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/CURRENT.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "batch_next": 0,
3
  "bytes": 67597675,
4
- "epoch_index": 1,
5
- "global_step": 3,
6
- "path": "checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt",
7
  "remote_verified": false,
8
- "sha256": "1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1",
9
- "timestamp_utc": "2026-11-05T02:33:27Z"
10
  }
 
1
  {
2
  "batch_next": 0,
3
  "bytes": 67597675,
4
+ "epoch_index": 14,
5
+ "global_step": 42,
6
+ "path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt",
7
  "remote_verified": false,
8
+ "sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92",
9
+ "timestamp_utc": "2026-11-05T02:43:54Z"
10
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/PREVIOUS.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "batch_next": 0,
3
- "bytes": 22535319,
4
- "epoch_index": 0,
5
- "global_step": 0,
6
- "path": "checkpoints/history/step_0000000_INITIAL.pt",
7
  "remote_verified": false,
8
- "sha256": "58bfec804840daed0a7666230a61272a85ca25d942ce1fdbb878e288d310f32a",
9
- "timestamp_utc": "2026-11-05T02:32:27Z"
10
  }
 
1
  {
2
  "batch_next": 0,
3
+ "bytes": 67597675,
4
+ "epoch_index": 13,
5
+ "global_step": 39,
6
+ "path": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt",
7
  "remote_verified": false,
8
+ "sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c",
9
+ "timestamp_utc": "2026-11-05T02:43:05Z"
10
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c
3
+ size 67597675
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92
3
+ size 67597675
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/finetuning/progress.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
- "best_epoch": 1,
3
- "best_validation_accuracy": 0.5385185185185185,
4
- "best_validation_macro_f1": 0.4921045345099603,
5
- "completed_epochs": 1,
6
- "global_step": 3,
7
- "latest_validation_accuracy": 0.5385185185185185,
8
- "latest_validation_macro_f1": 0.4921045345099603
9
  }
 
1
  {
2
+ "best_epoch": 14,
3
+ "best_validation_accuracy": 0.9007407407407407,
4
+ "best_validation_macro_f1": 0.8976430827821179,
5
+ "completed_epochs": 14,
6
+ "global_step": 42,
7
+ "latest_validation_accuracy": 0.9007407407407407,
8
+ "latest_validation_macro_f1": 0.8976430827821179
9
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/epoch_metrics.csv CHANGED
@@ -1,2 +1,15 @@
1
  timestamp_utc,epoch,global_step,train_loss_mean,validation_accuracy,validation_macro_f1,best_validation_accuracy,best_epoch,duration_seconds,checkpoint,checkpoint_sha256
2
  2026-11-05T02:33:27Z,1,3,2.0959319273630777,0.5385185185185185,0.4921045345099603,0.5385185185185185,1,49.75665736198425,checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt,1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  timestamp_utc,epoch,global_step,train_loss_mean,validation_accuracy,validation_macro_f1,best_validation_accuracy,best_epoch,duration_seconds,checkpoint,checkpoint_sha256
2
  2026-11-05T02:33:27Z,1,3,2.0959319273630777,0.5385185185185185,0.4921045345099603,0.5385185185185185,1,49.75665736198425,checkpoints/history/step_0000003_epoch_001_20261105T023238Z.pt,1487f24f1451ece96a10a5fd7cbe7559dd1dcde38306ea0e4e28f594ec9987c1
3
+ 2026-11-05T02:34:17Z,2,6,1.6399901310602825,0.6287037037037037,0.5767701775459999,0.6287037037037037,2,49.57090616226196,checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt,1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca
4
+ 2026-11-05T02:35:05Z,3,9,1.3544566631317139,0.6751851851851852,0.6327180638698691,0.6751851851851852,3,48.011171102523804,checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt,5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a
5
+ 2026-11-05T02:35:53Z,4,12,1.0348247090975444,0.7431481481481481,0.716856644247706,0.7431481481481481,4,47.63766622543335,checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt,65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973
6
+ 2026-11-05T02:36:40Z,5,15,0.8252936999003092,0.7922222222222223,0.7708581649718522,0.7922222222222223,5,47.327789306640625,checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt,2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f
7
+ 2026-11-05T02:37:27Z,6,18,0.6310902635256449,0.8362962962962963,0.8248050745115829,0.8362962962962963,6,47.18750858306885,checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt,9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482
8
+ 2026-11-05T02:38:15Z,7,21,0.4793592691421509,0.8651851851851852,0.8583025469404868,0.8651851851851852,7,47.810460567474365,checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt,f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd
9
+ 2026-11-05T02:39:02Z,8,24,0.35045451919237774,0.8751851851851852,0.8706107374650793,0.8751851851851852,8,46.78648495674133,checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt,73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3
10
+ 2026-11-05T02:39:49Z,9,27,0.24601457516352335,0.8855555555555555,0.8808243497096742,0.8855555555555555,9,47.03523898124695,checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt,7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd
11
+ 2026-11-05T02:40:38Z,10,30,0.20679345230261484,0.8822222222222222,0.8777383631501999,0.8855555555555555,9,48.99299097061157,checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt,317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2
12
+ 2026-11-05T02:41:27Z,11,33,0.15470425287882486,0.8905555555555555,0.8871090790401457,0.8905555555555555,11,49.47192049026489,checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt,a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004
13
+ 2026-11-05T02:42:17Z,12,36,0.12127700696388881,0.8931481481481481,0.8896603607977387,0.8931481481481481,12,49.385294914245605,checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt,31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7
14
+ 2026-11-05T02:43:05Z,13,39,0.0904170572757721,0.8987037037037037,0.8954650689040694,0.8987037037037037,13,48.701061725616455,checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt,48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c
15
+ 2026-11-05T02:43:54Z,14,42,0.08223462104797363,0.9007407407407407,0.8976430827821179,0.9007407407407407,14,48.82230854034424,checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt,ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/events.jsonl CHANGED
@@ -9,3 +9,68 @@
9
  {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.339874267578125, "epoch": 2, "global_step": 4, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.7395637035369873, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6463867425918579, "loss_real": 1.3266286849975586, "loss_reg_real": 2.305392026901245, "loss_reg_synthetic": 2.1597659587860107, "loss_synthetic": 1.336653470993042, "loss_total": 1.7940089702606201, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6243250370025635, "timestamp_utc": "2026-11-05T02:33:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.679715196440473}
10
  {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3444061279296875, "epoch": 2, "global_step": 5, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.4725263118743896, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6717017292976379, "loss_real": 1.2274569272994995, "loss_reg_real": 2.1863515377044678, "loss_reg_synthetic": 2.067314863204956, "loss_synthetic": 1.4349440336227417, "loss_total": 1.710416316986084, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.548011302947998, "timestamp_utc": "2026-11-05T02:33:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.144169280544894}
11
  {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.345935821533203, "epoch": 2, "global_step": 6, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 4.890103340148926, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5022993087768555, "loss_real": 0.9928121566772461, "loss_reg_real": 1.8100521564483643, "loss_reg_synthetic": 1.9769306182861328, "loss_synthetic": 1.3028115034103394, "loss_total": 1.415545105934143, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.50909423828125, "timestamp_utc": "2026-11-05T02:33:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.29030686140477}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.339874267578125, "epoch": 2, "global_step": 4, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.7395637035369873, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6463867425918579, "loss_real": 1.3266286849975586, "loss_reg_real": 2.305392026901245, "loss_reg_synthetic": 2.1597659587860107, "loss_synthetic": 1.336653470993042, "loss_total": 1.7940089702606201, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6243250370025635, "timestamp_utc": "2026-11-05T02:33:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.679715196440473}
10
  {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3444061279296875, "epoch": 2, "global_step": 5, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 3.4725263118743896, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.6717017292976379, "loss_real": 1.2274569272994995, "loss_reg_real": 2.1863515377044678, "loss_reg_synthetic": 2.067314863204956, "loss_synthetic": 1.4349440336227417, "loss_total": 1.710416316986084, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.548011302947998, "timestamp_utc": "2026-11-05T02:33:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.144169280544894}
11
  {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.345935821533203, "epoch": 2, "global_step": 6, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199361324310303, "gradient_norm": 4.890103340148926, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5022993087768555, "loss_real": 0.9928121566772461, "loss_reg_real": 1.8100521564483643, "loss_reg_synthetic": 1.9769306182861328, "loss_synthetic": 1.3028115034103394, "loss_total": 1.415545105934143, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.50909423828125, "timestamp_utc": "2026-11-05T02:33:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.29030686140477}
12
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 2, "global_step": 6, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt", "remote_verified": false, "sha256": "1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca", "timestamp_utc": "2026-11-05T02:34:17Z"}
13
+ {"arm": "spc_k0", "best_epoch": 2, "best_validation_accuracy": 0.6287037037037037, "checkpoint": "checkpoints/history/step_0000006_epoch_002_20261105T023238Z.pt", "checkpoint_sha256": "1f62a1bdf132a8bd7288fc30d9b3bb2e224f0654e2e431255cadef0a04e9c1ca", "duration_seconds": 49.57090616226196, "epoch": 2, "global_step": 6, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:34:17Z", "train_loss_mean": 1.6399901310602825, "validation_accuracy": 0.6287037037037037, "validation_macro_f1": 0.5767701775459999}
14
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348064422607422, "epoch": 3, "global_step": 7, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 3.2753567695617676, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.532727837562561, "loss_real": 0.8864139318466187, "loss_reg_real": 1.7116870880126953, "loss_reg_synthetic": 1.9229612350463867, "loss_synthetic": 1.1026084423065186, "loss_total": 1.2671351432800293, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6091248989105225, "timestamp_utc": "2026-11-05T02:34:22Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.770998358116934}
15
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.3471527099609375, "epoch": 3, "global_step": 8, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 5.345407485961914, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5581454634666443, "loss_real": 0.9600490927696228, "loss_reg_real": 1.8062270879745483, "loss_reg_synthetic": 1.8906059265136719, "loss_synthetic": 1.2284905910491943, "loss_total": 1.371622920036316, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.7241575717926025, "timestamp_utc": "2026-11-05T02:34:26Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.094777863522836}
16
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.348278045654297, "epoch": 3, "global_step": 9, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 8.87842082977295, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.7060779929161072, "loss_real": 1.022498607635498, "loss_reg_real": 1.7766468524932861, "loss_reg_synthetic": 1.8551563024520874, "loss_synthetic": 1.1166820526123047, "loss_total": 1.4246119260787964, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.492733001708984, "timestamp_utc": "2026-11-05T02:34:31Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.367840012634336}
17
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 3, "global_step": 9, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt", "remote_verified": false, "sha256": "5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a", "timestamp_utc": "2026-11-05T02:35:05Z"}
18
+ {"arm": "spc_k0", "best_epoch": 3, "best_validation_accuracy": 0.6751851851851852, "checkpoint": "checkpoints/history/step_0000009_epoch_003_20261105T023238Z.pt", "checkpoint_sha256": "5db4504c7a001fcaedd81be2a2c2f28fc1166bc39b8955d1f251e3aac67d338a", "duration_seconds": 48.011171102523804, "epoch": 3, "global_step": 9, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:35:05Z", "train_loss_mean": 1.3544566631317139, "validation_accuracy": 0.6751851851851852, "validation_macro_f1": 0.6327180638698691}
19
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348358154296875, "epoch": 4, "global_step": 10, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 5.862489223480225, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5491296648979187, "loss_real": 0.8255103826522827, "loss_reg_real": 1.4897865056991577, "loss_reg_synthetic": 1.6701234579086304, "loss_synthetic": 1.0721659660339355, "loss_total": 1.1878503561019897, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.607059717178345, "timestamp_utc": "2026-11-05T02:35:10Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.783447113291448}
20
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.348392486572266, "epoch": 4, "global_step": 11, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 3.9853928089141846, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5843984484672546, "loss_real": 0.6818434000015259, "loss_reg_real": 1.5178686380386353, "loss_reg_synthetic": 1.6483911275863647, "loss_synthetic": 1.025932788848877, "loss_total": 1.0391523838043213, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.588322877883911, "timestamp_utc": "2026-11-05T02:35:14Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.896903379875553}
21
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.3484649658203125, "epoch": 4, "global_step": 12, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199391841888428, "gradient_norm": 4.246176242828369, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5330894589424133, "loss_real": 0.5594723224639893, "loss_reg_real": 1.2257851362228394, "loss_reg_synthetic": 1.715860366821289, "loss_synthetic": 0.9067077040672302, "loss_total": 0.877471387386322, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.472378730773926, "timestamp_utc": "2026-11-05T02:35:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.465087323538814}
22
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 4, "global_step": 12, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt", "remote_verified": false, "sha256": "65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973", "timestamp_utc": "2026-11-05T02:35:53Z"}
23
+ {"arm": "spc_k0", "best_epoch": 4, "best_validation_accuracy": 0.7431481481481481, "checkpoint": "checkpoints/history/step_0000012_epoch_004_20261105T023238Z.pt", "checkpoint_sha256": "65d65c0c03f3c98174a26d7e6f1f0c7dae1f491d914906dae01ce8a90d9ac973", "duration_seconds": 47.63766622543335, "epoch": 4, "global_step": 12, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:35:53Z", "train_loss_mean": 1.0348247090975444, "validation_accuracy": 0.7431481481481481, "validation_macro_f1": 0.716856644247706}
24
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.348529815673828, "epoch": 5, "global_step": 13, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.1963372230529785, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5366214513778687, "loss_real": 0.6004937887191772, "loss_reg_real": 1.2908483743667603, "loss_reg_synthetic": 1.5194084644317627, "loss_synthetic": 0.875447690486908, "loss_total": 0.9112675786018372, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.4592626094818115, "timestamp_utc": "2026-11-05T02:35:57Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.70429737146031}
25
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.348590850830078, "epoch": 5, "global_step": 14, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.359305381774902, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5021079182624817, "loss_real": 0.46975645422935486, "loss_reg_real": 1.1514276266098022, "loss_reg_synthetic": 1.4126462936401367, "loss_synthetic": 0.8143174648284912, "loss_total": 0.7571407556533813, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.563748598098755, "timestamp_utc": "2026-11-05T02:36:02Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.047118996283988}
26
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.348659515380859, "epoch": 5, "global_step": 15, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.185586929321289, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.5958570837974548, "loss_real": 0.5067290663719177, "loss_reg_real": 1.0209497213363647, "loss_reg_synthetic": 1.5450220108032227, "loss_synthetic": 0.8470979928970337, "loss_total": 0.8074727654457092, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.366473197937012, "timestamp_utc": "2026-11-05T02:36:06Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.985706919111802}
27
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 5, "global_step": 15, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt", "remote_verified": false, "sha256": "2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f", "timestamp_utc": "2026-11-05T02:36:40Z"}
28
+ {"arm": "spc_k0", "best_epoch": 5, "best_validation_accuracy": 0.7922222222222223, "checkpoint": "checkpoints/history/step_0000015_epoch_005_20261105T023238Z.pt", "checkpoint_sha256": "2f0414dec08dcf2d012b6d8d1b56282c683685c79e7fc1410d5959c13fa0705f", "duration_seconds": 47.327789306640625, "epoch": 5, "global_step": 15, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:36:40Z", "train_loss_mean": 0.8252936999003092, "validation_accuracy": 0.7922222222222223, "validation_macro_f1": 0.7708581649718522}
29
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352695465087891, "epoch": 6, "global_step": 16, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.249288082122803, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.47728967666625977, "loss_real": 0.48116928339004517, "loss_reg_real": 1.0561646223068237, "loss_reg_synthetic": 1.4124089479446411, "loss_synthetic": 0.7735217809677124, "loss_total": 0.7540974020957947, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.513085842132568, "timestamp_utc": "2026-11-05T02:36:45Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.361968834059706}
30
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352790832519531, "epoch": 6, "global_step": 17, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.7728734016418457, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.40542393922805786, "loss_real": 0.31128787994384766, "loss_reg_real": 0.8898597955703735, "loss_reg_synthetic": 1.2429455518722534, "loss_synthetic": 0.6802616119384766, "loss_total": 0.5483358502388, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.580164432525635, "timestamp_utc": "2026-11-05T02:36:49Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.94659490629185}
31
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352806091308594, "epoch": 6, "global_step": 18, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 6.730944633483887, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.4384499490261078, "loss_real": 0.32776927947998047, "loss_reg_real": 0.7986207008361816, "loss_reg_synthetic": 1.2806344032287598, "loss_synthetic": 0.8083286285400391, "loss_total": 0.5908375382423401, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.276374340057373, "timestamp_utc": "2026-11-05T02:36:53Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 22.448923402414774}
32
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 6, "global_step": 18, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt", "remote_verified": false, "sha256": "9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482", "timestamp_utc": "2026-11-05T02:37:27Z"}
33
+ {"arm": "spc_k0", "best_epoch": 6, "best_validation_accuracy": 0.8362962962962963, "checkpoint": "checkpoints/history/step_0000018_epoch_006_20261105T023238Z.pt", "checkpoint_sha256": "9ea7d0ca1c85279a5d8df29a162a3c40eaa0d02f01f53200b38d9a20f87b8482", "duration_seconds": 47.18750858306885, "epoch": 6, "global_step": 18, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:37:27Z", "train_loss_mean": 0.6310902635256449, "validation_accuracy": 0.8362962962962963, "validation_macro_f1": 0.8248050745115829}
34
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352817535400391, "epoch": 7, "global_step": 19, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.4259095191955566, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3979944884777069, "loss_real": 0.27780306339263916, "loss_reg_real": 0.7724456191062927, "loss_reg_synthetic": 1.2027769088745117, "loss_synthetic": 0.687814474105835, "loss_total": 0.5101187825202942, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.736155033111572, "timestamp_utc": "2026-11-05T02:37:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.026142325393053}
35
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352817535400391, "epoch": 7, "global_step": 20, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.5623104572296143, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.280882865190506, "loss_real": 0.20820730924606323, "loss_reg_real": 0.7042987942695618, "loss_reg_synthetic": 1.0282939672470093, "loss_synthetic": 0.46521008014678955, "loss_total": 0.37807542085647583, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.497614622116089, "timestamp_utc": "2026-11-05T02:37:36Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.459530385415082}
36
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.35284423828125, "epoch": 7, "global_step": 21, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.640958309173584, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3802229166030884, "loss_real": 0.3616764545440674, "loss_reg_real": 0.5518221259117126, "loss_reg_synthetic": 1.0541349649429321, "loss_synthetic": 0.5351462960243225, "loss_total": 0.5498836040496826, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.428439140319824, "timestamp_utc": "2026-11-05T02:37:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.678066912096444}
37
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 7, "global_step": 21, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt", "remote_verified": false, "sha256": "f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd", "timestamp_utc": "2026-11-05T02:38:15Z"}
38
+ {"arm": "spc_k0", "best_epoch": 7, "best_validation_accuracy": 0.8651851851851852, "checkpoint": "checkpoints/history/step_0000021_epoch_007_20261105T023238Z.pt", "checkpoint_sha256": "f3ee5387db65af718886a47a3d1407e9726b80791fe5306fec85bf96db6fadbd", "duration_seconds": 47.810460567474365, "epoch": 7, "global_step": 21, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:38:15Z", "train_loss_mean": 0.4793592691421509, "validation_accuracy": 0.8651851851851852, "validation_macro_f1": 0.8583025469404868}
39
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352836608886719, "epoch": 8, "global_step": 22, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.94069766998291, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.38374873995780945, "loss_real": 0.19556286931037903, "loss_reg_real": 0.5717012286186218, "loss_reg_synthetic": 1.051354169845581, "loss_synthetic": 0.4724317193031311, "loss_total": 0.37231922149658203, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.457226276397705, "timestamp_utc": "2026-11-05T02:38:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.717411246944497}
40
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352851867675781, "epoch": 8, "global_step": 23, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.106137752532959, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.35716506838798523, "loss_real": 0.144283726811409, "loss_reg_real": 0.4893544316291809, "loss_reg_synthetic": 0.9479215741157532, "loss_synthetic": 0.4486972689628601, "loss_total": 0.30827227234840393, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.544239521026611, "timestamp_utc": "2026-11-05T02:38:24Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.167529331966836}
41
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352886199951172, "epoch": 8, "global_step": 24, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.545416355133057, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.40896984934806824, "loss_real": 0.1848861277103424, "loss_reg_real": 0.44575411081314087, "loss_reg_synthetic": 1.0729613304138184, "loss_synthetic": 0.5284978151321411, "loss_total": 0.3707720637321472, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.430412530899048, "timestamp_utc": "2026-11-05T02:38:28Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.668411085980534}
42
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 8, "global_step": 24, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt", "remote_verified": false, "sha256": "73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3", "timestamp_utc": "2026-11-05T02:39:02Z"}
43
+ {"arm": "spc_k0", "best_epoch": 8, "best_validation_accuracy": 0.8751851851851852, "checkpoint": "checkpoints/history/step_0000024_epoch_008_20261105T023238Z.pt", "checkpoint_sha256": "73a0bcf29ce5e4f527daba290f5a9c7b40db30b1a12b40108d6a5a84833282e3", "duration_seconds": 46.78648495674133, "epoch": 8, "global_step": 24, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:39:02Z", "train_loss_mean": 0.35045451919237774, "validation_accuracy": 0.8751851851851852, "validation_macro_f1": 0.8706107374650793}
44
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.3529052734375, "epoch": 9, "global_step": 25, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.5497570037841797, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.28275266289711, "loss_real": 0.10441966354846954, "loss_reg_real": 0.378287136554718, "loss_reg_synthetic": 0.8491285443305969, "loss_synthetic": 0.3608815670013428, "loss_total": 0.23698531091213226, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.535761594772339, "timestamp_utc": "2026-11-05T02:39:06Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.220178094793503}
45
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352931976318359, "epoch": 9, "global_step": 26, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 6.295091152191162, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3139715790748596, "loss_real": 0.13280969858169556, "loss_reg_real": 0.3783737123012543, "loss_reg_synthetic": 0.8116950392723083, "loss_synthetic": 0.398764044046402, "loss_total": 0.27532851696014404, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.540703773498535, "timestamp_utc": "2026-11-05T02:39:11Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 28.189462776026495}
46
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.352973937988281, "epoch": 9, "global_step": 27, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 5.602907180786133, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.267505943775177, "loss_real": 0.10692095756530762, "loss_reg_real": 0.3405652940273285, "loss_reg_synthetic": 0.7080139517784119, "loss_synthetic": 0.3213772773742676, "loss_total": 0.22572989761829376, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.302645921707153, "timestamp_utc": "2026-11-05T02:39:15Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 22.311852229269714}
47
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 9, "global_step": 27, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt", "remote_verified": false, "sha256": "7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd", "timestamp_utc": "2026-11-05T02:39:49Z"}
48
+ {"arm": "spc_k0", "best_epoch": 9, "best_validation_accuracy": 0.8855555555555555, "checkpoint": "checkpoints/history/step_0000027_epoch_009_20261105T023238Z.pt", "checkpoint_sha256": "7cb382b66a10067ab9c1bf2188f926cdd331079a3302d3088910926ab5da50cd", "duration_seconds": 47.03523898124695, "epoch": 9, "global_step": 27, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:39:49Z", "train_loss_mean": 0.24601457516352335, "validation_accuracy": 0.8855555555555555, "validation_macro_f1": 0.8808243497096742}
49
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.352973937988281, "epoch": 10, "global_step": 28, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.6516027450561523, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.26639851927757263, "loss_real": 0.07133440673351288, "loss_reg_real": 0.26976266503334045, "loss_reg_synthetic": 0.7569214105606079, "loss_synthetic": 0.29729902744293213, "loss_total": 0.18336300551891327, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.613707780838013, "timestamp_utc": "2026-11-05T02:39:53Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.74341290786099}
50
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.352993011474609, "epoch": 10, "global_step": 29, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.791995525360107, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.38612720370292664, "loss_real": 0.10022659599781036, "loss_reg_real": 0.32152873277664185, "loss_reg_synthetic": 0.8502084612846375, "loss_synthetic": 0.44544264674186707, "loss_total": 0.2577931582927704, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.586455821990967, "timestamp_utc": "2026-11-05T02:39:58Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.90825966016513}
51
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.353000640869141, "epoch": 10, "global_step": 30, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.241050958633423, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.3099075257778168, "loss_real": 0.05060635507106781, "loss_reg_real": 0.23522202670574188, "loss_reg_synthetic": 0.75429767370224, "loss_synthetic": 0.36566129326820374, "loss_total": 0.1792241930961609, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.429757595062256, "timestamp_utc": "2026-11-05T02:40:02Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.671614741856054}
52
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 10, "global_step": 30, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt", "remote_verified": false, "sha256": "317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2", "timestamp_utc": "2026-11-05T02:40:38Z"}
53
+ {"arm": "spc_k0", "best_epoch": 9, "best_validation_accuracy": 0.8855555555555555, "checkpoint": "checkpoints/history/step_0000030_epoch_010_20261105T023238Z.pt", "checkpoint_sha256": "317eacb020462159d9a466a19f193a07518f0052256903359b1025411880e0a2", "duration_seconds": 48.99299097061157, "epoch": 10, "global_step": 30, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:40:38Z", "train_loss_mean": 0.20679345230261484, "validation_accuracy": 0.8822222222222222, "validation_macro_f1": 0.8777383631501999}
54
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313625335693359, "epoch": 11, "global_step": 31, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.2985382080078125, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2581752836704254, "loss_real": 0.06394229829311371, "loss_reg_real": 0.23884811997413635, "loss_reg_synthetic": 0.6717081069946289, "loss_synthetic": 0.3720165193080902, "loss_total": 0.1871512085199356, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.6319260597229, "timestamp_utc": "2026-11-05T02:40:42Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.634292592239145}
55
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313652038574219, "epoch": 11, "global_step": 32, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.4699645042419434, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.16946427524089813, "loss_real": 0.05497533455491066, "loss_reg_real": 0.21618342399597168, "loss_reg_synthetic": 0.5425635576248169, "loss_synthetic": 0.2134230136871338, "loss_total": 0.13410496711730957, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.8092944622039795, "timestamp_utc": "2026-11-05T02:40:47Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.615130557287774}
56
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.31365966796875, "epoch": 11, "global_step": 33, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.234697103500366, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.19439589977264404, "loss_real": 0.045638084411621094, "loss_reg_real": 0.1863006353378296, "loss_reg_synthetic": 0.6746764183044434, "loss_synthetic": 0.28416675329208374, "loss_total": 0.14285658299922943, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.476435422897339, "timestamp_utc": "2026-11-05T02:40:52Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.445634959671715}
57
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 11, "global_step": 33, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt", "remote_verified": false, "sha256": "a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004", "timestamp_utc": "2026-11-05T02:41:27Z"}
58
+ {"arm": "spc_k0", "best_epoch": 11, "best_validation_accuracy": 0.8905555555555555, "checkpoint": "checkpoints/history/step_0000033_epoch_011_20261105T023238Z.pt", "checkpoint_sha256": "a84daefcc11ac5b07940e6d6466a5298ac579a02577e63c76968041ff8f9f004", "duration_seconds": 49.47192049026489, "epoch": 11, "global_step": 33, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:41:27Z", "train_loss_mean": 0.15470425287882486, "validation_accuracy": 0.8905555555555555, "validation_macro_f1": 0.8871090790401457}
59
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313663482666016, "epoch": 12, "global_step": 34, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.473418235778809, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2995773255825043, "loss_real": 0.041587233543395996, "loss_reg_real": 0.1665458232164383, "loss_reg_synthetic": 0.7011368870735168, "loss_synthetic": 0.2889729142189026, "loss_total": 0.15002411603927612, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.786452054977417, "timestamp_utc": "2026-11-05T02:41:32Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.742146067647997}
60
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.31365966796875, "epoch": 12, "global_step": 35, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.8804813623428345, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.1297117918729782, "loss_real": 0.03432588279247284, "loss_reg_real": 0.16799309849739075, "loss_reg_synthetic": 0.44184446334838867, "loss_synthetic": 0.16249723732471466, "loss_total": 0.09535311907529831, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.679177761077881, "timestamp_utc": "2026-11-05T02:41:37Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.35523344821897}
61
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.3136749267578125, "epoch": 12, "global_step": 36, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.603003978729248, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.1828250139951706, "loss_real": 0.041674211621284485, "loss_reg_real": 0.14130644500255585, "loss_reg_synthetic": 0.5113574862480164, "loss_synthetic": 0.21308833360671997, "loss_total": 0.11845378577709198, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.5488574504852295, "timestamp_utc": "2026-11-05T02:41:41Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.10420057013649}
62
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 12, "global_step": 36, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt", "remote_verified": false, "sha256": "31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7", "timestamp_utc": "2026-11-05T02:42:17Z"}
63
+ {"arm": "spc_k0", "best_epoch": 12, "best_validation_accuracy": 0.8931481481481481, "checkpoint": "checkpoints/history/step_0000036_epoch_012_20261105T023238Z.pt", "checkpoint_sha256": "31cc2665d3c37d93ad0dfb55fa8dd76760b2d47dca1bd7a91b799d651a8c7be7", "duration_seconds": 49.385294914245605, "epoch": 12, "global_step": 36, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:42:17Z", "train_loss_mean": 0.12127700696388881, "validation_accuracy": 0.8931481481481481, "validation_macro_f1": 0.8896603607977387}
64
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313690185546875, "epoch": 13, "global_step": 37, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.5368391275405884, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.10143797844648361, "loss_real": 0.03273775428533554, "loss_reg_real": 0.13649021089076996, "loss_reg_synthetic": 0.3714805841445923, "loss_synthetic": 0.1622830331325531, "loss_total": 0.08822737634181976, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.575104475021362, "timestamp_utc": "2026-11-05T02:42:21Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.97750318027488}
65
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313724517822266, "epoch": 13, "global_step": 38, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.45637583732605, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.20935462415218353, "loss_real": 0.0280852522701025, "loss_reg_real": 0.12176745384931564, "loss_reg_synthetic": 0.5415651798248291, "loss_synthetic": 0.21965403854846954, "loss_total": 0.10865352302789688, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.667818307876587, "timestamp_utc": "2026-11-05T02:42:26Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.421804268604408}
66
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313724517822266, "epoch": 13, "global_step": 39, "gpu_memory_gib": 4.20821475982666, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.1801849603652954, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.11240734905004501, "loss_real": 0.02421887405216694, "loss_reg_real": 0.1274520456790924, "loss_reg_synthetic": 0.36493992805480957, "loss_synthetic": 0.13256892561912537, "loss_total": 0.07437027245759964, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.467528581619263, "timestamp_utc": "2026-11-05T02:42:30Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.488390783882718}
67
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 13, "global_step": 39, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt", "remote_verified": false, "sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c", "timestamp_utc": "2026-11-05T02:43:05Z"}
68
+ {"arm": "spc_k0", "best_epoch": 13, "best_validation_accuracy": 0.8987037037037037, "checkpoint": "checkpoints/history/step_0000039_epoch_013_20261105T023238Z.pt", "checkpoint_sha256": "48a9345edb44dd2839cde9b6c310ebe77f29f8d94f5f9e5016e51ba9cd0c555c", "duration_seconds": 48.701061725616455, "epoch": 13, "global_step": 39, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:43:05Z", "train_loss_mean": 0.0904170572757721, "validation_accuracy": 0.8987037037037037, "validation_macro_f1": 0.8954650689040694}
69
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313732147216797, "epoch": 14, "global_step": 40, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.9312450885772705, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.2756049335002899, "loss_real": 0.017928674817085266, "loss_reg_real": 0.10229283571243286, "loss_reg_synthetic": 0.6242587566375732, "loss_synthetic": 0.24642586708068848, "loss_total": 0.11135122925043106, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.674060106277466, "timestamp_utc": "2026-11-05T02:43:10Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.38518484777088}
70
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313728332519531, "epoch": 14, "global_step": 41, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.085524082183838, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.11426927894353867, "loss_real": 0.020908314734697342, "loss_reg_real": 0.10161476582288742, "loss_reg_synthetic": 0.3838481903076172, "loss_synthetic": 0.14373445510864258, "loss_total": 0.07282369583845139, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.7962212562561035, "timestamp_utc": "2026-11-05T02:43:15Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.687676226996228}
71
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313732147216797, "epoch": 14, "global_step": 42, "gpu_memory_gib": 4.20736026763916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 1.676707148551941, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.06848233938217163, "loss_real": 0.02631022036075592, "loss_reg_real": 0.09765395522117615, "loss_reg_synthetic": 0.27334070205688477, "loss_synthetic": 0.09998758137226105, "loss_total": 0.06252893805503845, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.421590089797974, "timestamp_utc": "2026-11-05T02:43:19Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 21.71164627438052}
72
+ {"arm": "spc_k0", "batch_next": 0, "bytes": 67597675, "epoch_index": 14, "global_step": 42, "kind": "CHECKPOINT_LOCAL", "path": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt", "remote_verified": false, "sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92", "timestamp_utc": "2026-11-05T02:43:54Z"}
73
+ {"arm": "spc_k0", "best_epoch": 14, "best_validation_accuracy": 0.9007407407407407, "checkpoint": "checkpoints/history/step_0000042_epoch_014_20261105T023238Z.pt", "checkpoint_sha256": "ecba8240fa40802af5d72139028a9148aa854f3644fee7d48ad9c8378a59ad92", "duration_seconds": 48.82230854034424, "epoch": 14, "global_step": 42, "kind": "VALIDATION_COMPLETED", "timestamp_utc": "2026-11-05T02:43:54Z", "train_loss_mean": 0.08223462104797363, "validation_accuracy": 0.9007407407407407, "validation_macro_f1": 0.8976430827821179}
74
+ {"arm": "spc_k0", "batch": 1, "cpu_rss_gib": 6.313716888427734, "epoch": 15, "global_step": 43, "gpu_memory_gib": 4.21419620513916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 4.288538455963135, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.16103018820285797, "loss_real": 0.01848646253347397, "loss_reg_real": 0.08879290521144867, "loss_reg_synthetic": 0.37610241770744324, "loss_synthetic": 0.19105781614780426, "loss_total": 0.08387480676174164, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.769833326339722, "timestamp_utc": "2026-11-05T02:43:59Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 26.83531923288916}
75
+ {"arm": "spc_k0", "batch": 2, "cpu_rss_gib": 6.313720703125, "epoch": 15, "global_step": 44, "gpu_memory_gib": 4.21370792388916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 3.8296756744384766, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.18699847161769867, "loss_real": 0.01577068120241165, "loss_reg_real": 0.08303209394216537, "loss_reg_synthetic": 0.38371822237968445, "loss_synthetic": 0.1664869785308838, "loss_total": 0.07876357436180115, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.5843305587768555, "timestamp_utc": "2026-11-05T02:44:04Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 27.92119773189996}
76
+ {"arm": "spc_k0", "batch": 3, "cpu_rss_gib": 6.313755035400391, "epoch": 15, "global_step": 45, "gpu_memory_gib": 4.20784854888916, "gpu_peak_memory_gib": 13.199567317962646, "gradient_norm": 2.7890217304229736, "kind": "TRAIN_STEP", "learning_rate": 0.0001, "loss_epsilon": 0.13382680714130402, "loss_real": 0.01956470124423504, "loss_reg_real": 0.1038849875330925, "loss_reg_synthetic": 0.36285677552223206, "loss_synthetic": 0.15807963907718658, "loss_total": 0.07597584277391434, "recovery_segment_id": "20261105T023238Z", "step_duration_seconds": 4.581945180892944, "timestamp_utc": "2026-11-05T02:44:08Z", "train_run_id": "experiment_E45_spc_k0", "training_throughput": 20.951800209292596}
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/status.json CHANGED
@@ -2,38 +2,38 @@
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
- "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
6
- "completed_batches": 40,
7
- "cpu_rss_gib": 6.345935821533203,
8
- "epoch": 2,
9
  "epoch_eta_seconds": 0.0,
10
  "epochs": 45,
11
- "eta_seconds": 1.7384650204330683,
12
- "evaluated": 5120,
13
- "global_step": 6,
14
- "gpu_memory_gib": 4.20736026763916,
15
- "gpu_peak_memory_gib": 13.199361324310303,
16
- "gradient_norm": 4.890103340148926,
17
- "latest_chunk": "validation/epoch_002/chunks/00040.npz",
18
  "learning_rate": 0.0001,
19
- "loss_epsilon": 0.5022993087768555,
20
- "loss_real": 0.9928121566772461,
21
- "loss_reg_real": 1.8100521564483643,
22
- "loss_reg_synthetic": 1.9769306182861328,
23
- "loss_synthetic": 1.3028115034103394,
24
- "loss_total": 1.415545105934143,
25
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
26
  "operation": "official_forward_backward_optimizer",
27
  "phase": "VALIDATING",
28
  "pid": 10002,
29
  "recovery_segment_id": "20261105T023238Z",
30
  "samples": 5400,
31
- "step_duration_seconds": 4.50909423828125,
32
- "throughput": 161.06162431168693,
33
- "timestamp_utc": "2026-11-05T02:33:41Z",
34
  "total_batches": 43,
35
  "train_run_id": "experiment_E45_spc_k0",
36
- "training_throughput": 21.29030686140477,
37
- "updated_unix": 1793846053.8395257,
38
- "updated_utc": "2026-11-05T02:34:13Z"
39
  }
 
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
+ "checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
6
+ "completed_batches": 12,
7
+ "cpu_rss_gib": 6.313755035400391,
8
+ "epoch": 15,
9
  "epoch_eta_seconds": 0.0,
10
  "epochs": 45,
11
+ "eta_seconds": 23.86163430660963,
12
+ "evaluated": 1536,
13
+ "global_step": 45,
14
+ "gpu_memory_gib": 4.20784854888916,
15
+ "gpu_peak_memory_gib": 13.199567317962646,
16
+ "gradient_norm": 2.7890217304229736,
17
+ "latest_chunk": "validation/epoch_015/chunks/00012.npz",
18
  "learning_rate": 0.0001,
19
+ "loss_epsilon": 0.13382680714130402,
20
+ "loss_real": 0.01956470124423504,
21
+ "loss_reg_real": 0.1038849875330925,
22
+ "loss_reg_synthetic": 0.36285677552223206,
23
+ "loss_synthetic": 0.15807963907718658,
24
+ "loss_total": 0.07597584277391434,
25
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
26
  "operation": "official_forward_backward_optimizer",
27
  "phase": "VALIDATING",
28
  "pid": 10002,
29
  "recovery_segment_id": "20261105T023238Z",
30
  "samples": 5400,
31
+ "step_duration_seconds": 4.581945180892944,
32
+ "throughput": 161.9335855352405,
33
+ "timestamp_utc": "2026-11-05T02:44:08Z",
34
  "total_batches": 43,
35
  "train_run_id": "experiment_E45_spc_k0",
36
+ "training_throughput": 20.951800209292596,
37
+ "updated_unix": 1793846658.5428693,
38
+ "updated_utc": "2026-11-05T02:44:18Z"
39
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/logs/train_steps.csv CHANGED
@@ -5,3 +5,42 @@ timestamp_utc,train_run_id,recovery_segment_id,epoch,batch,global_step,loss_tota
5
  2026-11-05T02:33:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,1,4,1.7940089702606201,1.3266286849975586,1.336653470993042,0.6463867425918579,2.305392026901245,2.1597659587860107,0.0001,4.6243250370025635,27.679715196440473,4.21370792388916,13.199361324310303,6.339874267578125,3.7395637035369873
6
  2026-11-05T02:33:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,2,5,1.710416316986084,1.2274569272994995,1.4349440336227417,0.6717017292976379,2.1863515377044678,2.067314863204956,0.0001,4.548011302947998,28.144169280544894,4.21419620513916,13.199361324310303,6.3444061279296875,3.4725263118743896
7
  2026-11-05T02:33:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,3,6,1.415545105934143,0.9928121566772461,1.3028115034103394,0.5022993087768555,1.8100521564483643,1.9769306182861328,0.0001,4.50909423828125,21.29030686140477,4.20736026763916,13.199361324310303,6.345935821533203,4.890103340148926
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  2026-11-05T02:33:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,1,4,1.7940089702606201,1.3266286849975586,1.336653470993042,0.6463867425918579,2.305392026901245,2.1597659587860107,0.0001,4.6243250370025635,27.679715196440473,4.21370792388916,13.199361324310303,6.339874267578125,3.7395637035369873
6
  2026-11-05T02:33:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,2,5,1.710416316986084,1.2274569272994995,1.4349440336227417,0.6717017292976379,2.1863515377044678,2.067314863204956,0.0001,4.548011302947998,28.144169280544894,4.21419620513916,13.199361324310303,6.3444061279296875,3.4725263118743896
7
  2026-11-05T02:33:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,2,3,6,1.415545105934143,0.9928121566772461,1.3028115034103394,0.5022993087768555,1.8100521564483643,1.9769306182861328,0.0001,4.50909423828125,21.29030686140477,4.20736026763916,13.199361324310303,6.345935821533203,4.890103340148926
8
+ 2026-11-05T02:34:22Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,1,7,1.2671351432800293,0.8864139318466187,1.1026084423065186,0.532727837562561,1.7116870880126953,1.9229612350463867,0.0001,4.6091248989105225,27.770998358116934,4.21419620513916,13.199391841888428,6.348064422607422,3.2753567695617676
9
+ 2026-11-05T02:34:26Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,2,8,1.371622920036316,0.9600490927696228,1.2284905910491943,0.5581454634666443,1.8062270879745483,1.8906059265136719,0.0001,4.7241575717926025,27.094777863522836,4.21370792388916,13.199391841888428,6.3471527099609375,5.345407485961914
10
+ 2026-11-05T02:34:31Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,3,3,9,1.4246119260787964,1.022498607635498,1.1166820526123047,0.7060779929161072,1.7766468524932861,1.8551563024520874,0.0001,4.492733001708984,21.367840012634336,4.20784854888916,13.199391841888428,6.348278045654297,8.87842082977295
11
+ 2026-11-05T02:35:10Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,1,10,1.1878503561019897,0.8255103826522827,1.0721659660339355,0.5491296648979187,1.4897865056991577,1.6701234579086304,0.0001,4.607059717178345,27.783447113291448,4.21370792388916,13.199391841888428,6.348358154296875,5.862489223480225
12
+ 2026-11-05T02:35:14Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,2,11,1.0391523838043213,0.6818434000015259,1.025932788848877,0.5843984484672546,1.5178686380386353,1.6483911275863647,0.0001,4.588322877883911,27.896903379875553,4.21419620513916,13.199391841888428,6.348392486572266,3.9853928089141846
13
+ 2026-11-05T02:35:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,4,3,12,0.877471387386322,0.5594723224639893,0.9067077040672302,0.5330894589424133,1.2257851362228394,1.715860366821289,0.0001,4.472378730773926,21.465087323538814,4.20736026763916,13.199391841888428,6.3484649658203125,4.246176242828369
14
+ 2026-11-05T02:35:57Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,1,13,0.9112675786018372,0.6004937887191772,0.875447690486908,0.5366214513778687,1.2908483743667603,1.5194084644317627,0.0001,4.4592626094818115,28.70429737146031,4.21419620513916,13.199567317962646,6.348529815673828,4.1963372230529785
15
+ 2026-11-05T02:36:02Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,2,14,0.7571407556533813,0.46975645422935486,0.8143174648284912,0.5021079182624817,1.1514276266098022,1.4126462936401367,0.0001,4.563748598098755,28.047118996283988,4.21370792388916,13.199567317962646,6.348590850830078,4.359305381774902
16
+ 2026-11-05T02:36:06Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,5,3,15,0.8074727654457092,0.5067290663719177,0.8470979928970337,0.5958570837974548,1.0209497213363647,1.5450220108032227,0.0001,4.366473197937012,21.985706919111802,4.20821475982666,13.199567317962646,6.348659515380859,5.185586929321289
17
+ 2026-11-05T02:36:45Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,1,16,0.7540974020957947,0.48116928339004517,0.7735217809677124,0.47728967666625977,1.0561646223068237,1.4124089479446411,0.0001,4.513085842132568,28.361968834059706,4.21370792388916,13.199567317962646,6.352695465087891,4.249288082122803
18
+ 2026-11-05T02:36:49Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,2,17,0.5483358502388,0.31128787994384766,0.6802616119384766,0.40542393922805786,0.8898597955703735,1.2429455518722534,0.0001,4.580164432525635,27.94659490629185,4.21419620513916,13.199567317962646,6.352790832519531,3.7728734016418457
19
+ 2026-11-05T02:36:53Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,6,3,18,0.5908375382423401,0.32776927947998047,0.8083286285400391,0.4384499490261078,0.7986207008361816,1.2806344032287598,0.0001,4.276374340057373,22.448923402414774,4.20736026763916,13.199567317962646,6.352806091308594,6.730944633483887
20
+ 2026-11-05T02:37:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,1,19,0.5101187825202942,0.27780306339263916,0.687814474105835,0.3979944884777069,0.7724456191062927,1.2027769088745117,0.0001,4.736155033111572,27.026142325393053,4.21419620513916,13.199567317962646,6.352817535400391,3.4259095191955566
21
+ 2026-11-05T02:37:36Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,2,20,0.37807542085647583,0.20820730924606323,0.46521008014678955,0.280882865190506,0.7042987942695618,1.0282939672470093,0.0001,4.497614622116089,28.459530385415082,4.21370792388916,13.199567317962646,6.352817535400391,3.5623104572296143
22
+ 2026-11-05T02:37:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,7,3,21,0.5498836040496826,0.3616764545440674,0.5351462960243225,0.3802229166030884,0.5518221259117126,1.0541349649429321,0.0001,4.428439140319824,21.678066912096444,4.20784854888916,13.199567317962646,6.35284423828125,5.640958309173584
23
+ 2026-11-05T02:38:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,1,22,0.37231922149658203,0.19556286931037903,0.4724317193031311,0.38374873995780945,0.5717012286186218,1.051354169845581,0.0001,4.457226276397705,28.717411246944497,4.21370792388916,13.199567317962646,6.352836608886719,2.94069766998291
24
+ 2026-11-05T02:38:24Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,2,23,0.30827227234840393,0.144283726811409,0.4486972689628601,0.35716506838798523,0.4893544316291809,0.9479215741157532,0.0001,4.544239521026611,28.167529331966836,4.21419620513916,13.199567317962646,6.352851867675781,3.106137752532959
25
+ 2026-11-05T02:38:28Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,8,3,24,0.3707720637321472,0.1848861277103424,0.5284978151321411,0.40896984934806824,0.44575411081314087,1.0729613304138184,0.0001,4.430412530899048,21.668411085980534,4.20736026763916,13.199567317962646,6.352886199951172,4.545416355133057
26
+ 2026-11-05T02:39:06Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,1,25,0.23698531091213226,0.10441966354846954,0.3608815670013428,0.28275266289711,0.378287136554718,0.8491285443305969,0.0001,4.535761594772339,28.220178094793503,4.21419620513916,13.199567317962646,6.3529052734375,2.5497570037841797
27
+ 2026-11-05T02:39:11Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,2,26,0.27532851696014404,0.13280969858169556,0.398764044046402,0.3139715790748596,0.3783737123012543,0.8116950392723083,0.0001,4.540703773498535,28.189462776026495,4.21370792388916,13.199567317962646,6.352931976318359,6.295091152191162
28
+ 2026-11-05T02:39:15Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,9,3,27,0.22572989761829376,0.10692095756530762,0.3213772773742676,0.267505943775177,0.3405652940273285,0.7080139517784119,0.0001,4.302645921707153,22.311852229269714,4.20821475982666,13.199567317962646,6.352973937988281,5.602907180786133
29
+ 2026-11-05T02:39:53Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,1,28,0.18336300551891327,0.07133440673351288,0.29729902744293213,0.26639851927757263,0.26976266503334045,0.7569214105606079,0.0001,4.613707780838013,27.74341290786099,4.21370792388916,13.199567317962646,6.352973937988281,2.6516027450561523
30
+ 2026-11-05T02:39:58Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,2,29,0.2577931582927704,0.10022659599781036,0.44544264674186707,0.38612720370292664,0.32152873277664185,0.8502084612846375,0.0001,4.586455821990967,27.90825966016513,4.21419620513916,13.199567317962646,6.352993011474609,4.791995525360107
31
+ 2026-11-05T02:40:02Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,10,3,30,0.1792241930961609,0.05060635507106781,0.36566129326820374,0.3099075257778168,0.23522202670574188,0.75429767370224,0.0001,4.429757595062256,21.671614741856054,4.20736026763916,13.199567317962646,6.353000640869141,3.241050958633423
32
+ 2026-11-05T02:40:42Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,1,31,0.1871512085199356,0.06394229829311371,0.3720165193080902,0.2581752836704254,0.23884811997413635,0.6717081069946289,0.0001,4.6319260597229,27.634292592239145,4.21419620513916,13.199567317962646,6.313625335693359,4.2985382080078125
33
+ 2026-11-05T02:40:47Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,2,32,0.13410496711730957,0.05497533455491066,0.2134230136871338,0.16946427524089813,0.21618342399597168,0.5425635576248169,0.0001,4.8092944622039795,26.615130557287774,4.21370792388916,13.199567317962646,6.313652038574219,2.4699645042419434
34
+ 2026-11-05T02:40:52Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,11,3,33,0.14285658299922943,0.045638084411621094,0.28416675329208374,0.19439589977264404,0.1863006353378296,0.6746764183044434,0.0001,4.476435422897339,21.445634959671715,4.20784854888916,13.199567317962646,6.31365966796875,2.234697103500366
35
+ 2026-11-05T02:41:32Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,1,34,0.15002411603927612,0.041587233543395996,0.2889729142189026,0.2995773255825043,0.1665458232164383,0.7011368870735168,0.0001,4.786452054977417,26.742146067647997,4.21370792388916,13.199567317962646,6.313663482666016,4.473418235778809
36
+ 2026-11-05T02:41:37Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,2,35,0.09535311907529831,0.03432588279247284,0.16249723732471466,0.1297117918729782,0.16799309849739075,0.44184446334838867,0.0001,4.679177761077881,27.35523344821897,4.21419620513916,13.199567317962646,6.31365966796875,1.8804813623428345
37
+ 2026-11-05T02:41:41Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,12,3,36,0.11845378577709198,0.041674211621284485,0.21308833360671997,0.1828250139951706,0.14130644500255585,0.5113574862480164,0.0001,4.5488574504852295,21.10420057013649,4.20736026763916,13.199567317962646,6.3136749267578125,3.603003978729248
38
+ 2026-11-05T02:42:21Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,1,37,0.08822737634181976,0.03273775428533554,0.1622830331325531,0.10143797844648361,0.13649021089076996,0.3714805841445923,0.0001,4.575104475021362,27.97750318027488,4.21419620513916,13.199567317962646,6.313690185546875,1.5368391275405884
39
+ 2026-11-05T02:42:26Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,2,38,0.10865352302789688,0.0280852522701025,0.21965403854846954,0.20935462415218353,0.12176745384931564,0.5415651798248291,0.0001,4.667818307876587,27.421804268604408,4.21370792388916,13.199567317962646,6.313724517822266,2.45637583732605
40
+ 2026-11-05T02:42:30Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,13,3,39,0.07437027245759964,0.02421887405216694,0.13256892561912537,0.11240734905004501,0.1274520456790924,0.36493992805480957,0.0001,4.467528581619263,21.488390783882718,4.20821475982666,13.199567317962646,6.313724517822266,1.1801849603652954
41
+ 2026-11-05T02:43:10Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,1,40,0.11135122925043106,0.017928674817085266,0.24642586708068848,0.2756049335002899,0.10229283571243286,0.6242587566375732,0.0001,4.674060106277466,27.38518484777088,4.21370792388916,13.199567317962646,6.313732147216797,3.9312450885772705
42
+ 2026-11-05T02:43:15Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,2,41,0.07282369583845139,0.020908314734697342,0.14373445510864258,0.11426927894353867,0.10161476582288742,0.3838481903076172,0.0001,4.7962212562561035,26.687676226996228,4.21419620513916,13.199567317962646,6.313728332519531,2.085524082183838
43
+ 2026-11-05T02:43:19Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,14,3,42,0.06252893805503845,0.02631022036075592,0.09998758137226105,0.06848233938217163,0.09765395522117615,0.27334070205688477,0.0001,4.421590089797974,21.71164627438052,4.20736026763916,13.199567317962646,6.313732147216797,1.676707148551941
44
+ 2026-11-05T02:43:59Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,1,43,0.08387480676174164,0.01848646253347397,0.19105781614780426,0.16103018820285797,0.08879290521144867,0.37610241770744324,0.0001,4.769833326339722,26.83531923288916,4.21419620513916,13.199567317962646,6.313716888427734,4.288538455963135
45
+ 2026-11-05T02:44:04Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,2,44,0.07876357436180115,0.01577068120241165,0.1664869785308838,0.18699847161769867,0.08303209394216537,0.38371822237968445,0.0001,4.5843305587768555,27.92119773189996,4.21370792388916,13.199567317962646,6.313720703125,3.8296756744384766
46
+ 2026-11-05T02:44:08Z,d45_e45_20261104T122347Z_E45,20261105T023238Z,15,3,45,0.07597584277391434,0.01956470124423504,0.15807963907718658,0.13382680714130402,0.1038849875330925,0.36285677552223206,0.0001,4.581945180892944,20.951800209292596,4.20784854888916,13.199567317962646,6.313755035400391,2.7890217304229736
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/live_telemetry.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "gpu_utilization": 39.0,
3
  "gpu_memory_mib": 15867.0,
4
- "timestamp_unix": 1793846050.848078
5
  }
 
1
  {
2
+ "gpu_utilization": 41.0,
3
  "gpu_memory_mib": 15867.0,
4
+ "timestamp_unix": 1793846655.7590516
5
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/manifests/worker_status.json CHANGED
@@ -2,38 +2,38 @@
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
- "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
6
- "completed_batches": 40,
7
- "cpu_rss_gib": 6.345935821533203,
8
- "epoch": 2,
9
  "epoch_eta_seconds": 0.0,
10
  "epochs": 45,
11
- "eta_seconds": 1.7384650204330683,
12
- "evaluated": 5120,
13
- "global_step": 6,
14
- "gpu_memory_gib": 4.20736026763916,
15
- "gpu_peak_memory_gib": 13.199361324310303,
16
- "gradient_norm": 4.890103340148926,
17
- "latest_chunk": "validation/epoch_002/chunks/00040.npz",
18
  "learning_rate": 0.0001,
19
- "loss_epsilon": 0.5022993087768555,
20
- "loss_real": 0.9928121566772461,
21
- "loss_reg_real": 1.8100521564483643,
22
- "loss_reg_synthetic": 1.9769306182861328,
23
- "loss_synthetic": 1.3028115034103394,
24
- "loss_total": 1.415545105934143,
25
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
26
  "operation": "official_forward_backward_optimizer",
27
  "phase": "VALIDATING",
28
  "pid": 10002,
29
  "recovery_segment_id": "20261105T023238Z",
30
  "samples": 5400,
31
- "step_duration_seconds": 4.50909423828125,
32
- "throughput": 161.06162431168693,
33
- "timestamp_utc": "2026-11-05T02:33:41Z",
34
  "total_batches": 43,
35
  "train_run_id": "experiment_E45_spc_k0",
36
- "training_throughput": 21.29030686140477,
37
- "updated_unix": 1793846053.8395257,
38
- "updated_utc": "2026-11-05T02:34:13Z"
39
  }
 
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
+ "checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
6
+ "completed_batches": 12,
7
+ "cpu_rss_gib": 6.313755035400391,
8
+ "epoch": 15,
9
  "epoch_eta_seconds": 0.0,
10
  "epochs": 45,
11
+ "eta_seconds": 23.86163430660963,
12
+ "evaluated": 1536,
13
+ "global_step": 45,
14
+ "gpu_memory_gib": 4.20784854888916,
15
+ "gpu_peak_memory_gib": 13.199567317962646,
16
+ "gradient_norm": 2.7890217304229736,
17
+ "latest_chunk": "validation/epoch_015/chunks/00012.npz",
18
  "learning_rate": 0.0001,
19
+ "loss_epsilon": 0.13382680714130402,
20
+ "loss_real": 0.01956470124423504,
21
+ "loss_reg_real": 0.1038849875330925,
22
+ "loss_reg_synthetic": 0.36285677552223206,
23
+ "loss_synthetic": 0.15807963907718658,
24
+ "loss_total": 0.07597584277391434,
25
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
26
  "operation": "official_forward_backward_optimizer",
27
  "phase": "VALIDATING",
28
  "pid": 10002,
29
  "recovery_segment_id": "20261105T023238Z",
30
  "samples": 5400,
31
+ "step_duration_seconds": 4.581945180892944,
32
+ "throughput": 161.9335855352405,
33
+ "timestamp_utc": "2026-11-05T02:44:08Z",
34
  "total_batches": 43,
35
  "train_run_id": "experiment_E45_spc_k0",
36
+ "training_throughput": 20.951800209292596,
37
+ "updated_unix": 1793846658.5428693,
38
+ "updated_utc": "2026-11-05T02:44:18Z"
39
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/state.json CHANGED
@@ -2,69 +2,69 @@
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
- "best_acc": 0.5385185185185185,
6
- "best_epoch": 1,
7
- "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
8
- "completed_batches": 37,
9
- "cpu_rss_gib": 6.345935821533203,
10
- "epoch": 2,
11
  "epoch_eta_seconds": 0.0,
12
  "epochs": 45,
13
- "eta_seconds": 4.0961752433229135,
14
- "evaluated": 4736,
15
  "evaluation_progress": {
16
  "arm": "spc_k0",
17
  "batch": 3,
18
  "batches": 3,
19
- "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
20
- "completed_batches": 37,
21
- "cpu_rss_gib": 6.345935821533203,
22
- "epoch": 2,
23
  "epoch_eta_seconds": 0.0,
24
  "epochs": 45,
25
- "eta_seconds": 4.0961752433229135,
26
- "evaluated": 4736,
27
- "global_step": 6,
28
- "gpu_memory_gib": 4.20736026763916,
29
- "gpu_peak_memory_gib": 13.199361324310303,
30
- "gradient_norm": 4.890103340148926,
31
- "latest_chunk": "validation/epoch_002/chunks/00037.npz",
32
  "learning_rate": 0.0001,
33
- "loss_epsilon": 0.5022993087768555,
34
- "loss_real": 0.9928121566772461,
35
- "loss_reg_real": 1.8100521564483643,
36
- "loss_reg_synthetic": 1.9769306182861328,
37
- "loss_synthetic": 1.3028115034103394,
38
- "loss_total": 1.415545105934143,
39
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
40
  "operation": "official_forward_backward_optimizer",
41
  "phase": "VALIDATING",
42
  "pid": 10002,
43
  "recovery_segment_id": "20261105T023238Z",
44
  "samples": 5400,
45
- "step_duration_seconds": 4.50909423828125,
46
- "throughput": 162.1024396068923,
47
- "timestamp_utc": "2026-11-05T02:33:41Z",
48
  "total_batches": 43,
49
  "train_run_id": "experiment_E45_spc_k0",
50
- "training_throughput": 21.29030686140477,
51
- "updated_unix": 1793846051.2666724,
52
- "updated_utc": "2026-11-05T02:34:11Z"
53
  },
54
  "experiment_id": "SPC_EUROSAT_HAIHT_PROMPTS_45E",
55
- "global_step": 6,
56
- "gpu_memory_gib": 4.20736026763916,
57
- "gpu_peak_memory_gib": 13.199361324310303,
58
- "gradient_norm": 4.890103340148926,
59
- "heartbeat": "2026-11-05T02:34:11Z",
60
- "latest_chunk": "validation/epoch_002/chunks/00037.npz",
61
  "learning_rate": 0.0001,
62
- "loss_epsilon": 0.5022993087768555,
63
- "loss_real": 0.9928121566772461,
64
- "loss_reg_real": 1.8100521564483643,
65
- "loss_reg_synthetic": 1.9769306182861328,
66
- "loss_synthetic": 1.3028115034103394,
67
- "loss_total": 1.415545105934143,
68
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
69
  "operation": "official_forward_backward_optimizer",
70
  "phase": "VALIDATING",
@@ -72,14 +72,14 @@
72
  "recovery_segment_id": "20261105T023238Z",
73
  "samples": 5400,
74
  "stage": "VALIDATING",
75
- "step_duration_seconds": 4.50909423828125,
76
- "throughput": 162.1024396068923,
77
- "timestamp_utc": "2026-11-05T02:33:41Z",
78
  "total_batches": 43,
79
  "total_epochs": 45,
80
  "train_run_id": "d45_e45_20261104T122347Z_E45",
81
- "training_throughput": 21.29030686140477,
82
- "updated_unix": 1793846051.2666724,
83
- "updated_utc": "2026-11-05T02:34:11Z",
84
  "worker_pid": 10002
85
  }
 
2
  "arm": "spc_k0",
3
  "batch": 3,
4
  "batches": 3,
5
+ "best_acc": 0.9007407407407407,
6
+ "best_epoch": 14,
7
+ "checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
8
+ "completed_batches": 9,
9
+ "cpu_rss_gib": 6.313755035400391,
10
+ "epoch": 15,
11
  "epoch_eta_seconds": 0.0,
12
  "epochs": 45,
13
+ "eta_seconds": 26.221441715955734,
14
+ "evaluated": 1152,
15
  "evaluation_progress": {
16
  "arm": "spc_k0",
17
  "batch": 3,
18
  "batches": 3,
19
+ "checkpoint_identity": "63761645c4519d8ae444de007a0eb1183e2abe292e1729c2f69023458747dc26",
20
+ "completed_batches": 9,
21
+ "cpu_rss_gib": 6.313755035400391,
22
+ "epoch": 15,
23
  "epoch_eta_seconds": 0.0,
24
  "epochs": 45,
25
+ "eta_seconds": 26.221441715955734,
26
+ "evaluated": 1152,
27
+ "global_step": 45,
28
+ "gpu_memory_gib": 4.20784854888916,
29
+ "gpu_peak_memory_gib": 13.199567317962646,
30
+ "gradient_norm": 2.7890217304229736,
31
+ "latest_chunk": "validation/epoch_015/chunks/00009.npz",
32
  "learning_rate": 0.0001,
33
+ "loss_epsilon": 0.13382680714130402,
34
+ "loss_real": 0.01956470124423504,
35
+ "loss_reg_real": 0.1038849875330925,
36
+ "loss_reg_synthetic": 0.36285677552223206,
37
+ "loss_synthetic": 0.15807963907718658,
38
+ "loss_total": 0.07597584277391434,
39
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
40
  "operation": "official_forward_backward_optimizer",
41
  "phase": "VALIDATING",
42
  "pid": 10002,
43
  "recovery_segment_id": "20261105T023238Z",
44
  "samples": 5400,
45
+ "step_duration_seconds": 4.581945180892944,
46
+ "throughput": 162.0048220847862,
47
+ "timestamp_utc": "2026-11-05T02:44:08Z",
48
  "total_batches": 43,
49
  "train_run_id": "experiment_E45_spc_k0",
50
+ "training_throughput": 20.951800209292596,
51
+ "updated_unix": 1793846656.168427,
52
+ "updated_utc": "2026-11-05T02:44:16Z"
53
  },
54
  "experiment_id": "SPC_EUROSAT_HAIHT_PROMPTS_45E",
55
+ "global_step": 45,
56
+ "gpu_memory_gib": 4.20784854888916,
57
+ "gpu_peak_memory_gib": 13.199567317962646,
58
+ "gradient_norm": 2.7890217304229736,
59
+ "heartbeat": "2026-11-05T02:44:16Z",
60
+ "latest_chunk": "validation/epoch_015/chunks/00009.npz",
61
  "learning_rate": 0.0001,
62
+ "loss_epsilon": 0.13382680714130402,
63
+ "loss_real": 0.01956470124423504,
64
+ "loss_reg_real": 0.1038849875330925,
65
+ "loss_reg_synthetic": 0.36285677552223206,
66
+ "loss_synthetic": 0.15807963907718658,
67
+ "loss_total": 0.07597584277391434,
68
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
69
  "operation": "official_forward_backward_optimizer",
70
  "phase": "VALIDATING",
 
72
  "recovery_segment_id": "20261105T023238Z",
73
  "samples": 5400,
74
  "stage": "VALIDATING",
75
+ "step_duration_seconds": 4.581945180892944,
76
+ "throughput": 162.0048220847862,
77
+ "timestamp_utc": "2026-11-05T02:44:08Z",
78
  "total_batches": 43,
79
  "total_epochs": 45,
80
  "train_run_id": "d45_e45_20261104T122347Z_E45",
81
+ "training_throughput": 20.951800209292596,
82
+ "updated_unix": 1793846656.168427,
83
+ "updated_utc": "2026-11-05T02:44:16Z",
84
  "worker_pid": 10002
85
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00041.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d7868fa5a6fc0d2266966f6e96db483e35b02b8d7c44d04f72bfa210332e6df
3
+ size 7747
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00042.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77142e529728ee7fa716220ce39fe0856bd2b63bffd7559078510d483569e218
3
+ size 7744
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/chunks/00043.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5cbea9f035b0cbbb43533318bea3bbe389b5010aa44e2c03973cf69700bb68d
3
+ size 2732
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/metrics.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy": 0.6287037037037037,
3
+ "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
4
+ "confusion_matrix": [
5
+ [
6
+ 477,
7
+ 2,
8
+ 30,
9
+ 19,
10
+ 5,
11
+ 37,
12
+ 16,
13
+ 2,
14
+ 9,
15
+ 3
16
+ ],
17
+ [
18
+ 1,
19
+ 544,
20
+ 23,
21
+ 1,
22
+ 0,
23
+ 3,
24
+ 0,
25
+ 1,
26
+ 6,
27
+ 21
28
+ ],
29
+ [
30
+ 8,
31
+ 226,
32
+ 19,
33
+ 51,
34
+ 3,
35
+ 168,
36
+ 2,
37
+ 15,
38
+ 103,
39
+ 5
40
+ ],
41
+ [
42
+ 16,
43
+ 0,
44
+ 0,
45
+ 286,
46
+ 84,
47
+ 30,
48
+ 2,
49
+ 34,
50
+ 48,
51
+ 0
52
+ ],
53
+ [
54
+ 0,
55
+ 0,
56
+ 0,
57
+ 1,
58
+ 333,
59
+ 1,
60
+ 3,
61
+ 162,
62
+ 0,
63
+ 0
64
+ ],
65
+ [
66
+ 118,
67
+ 25,
68
+ 0,
69
+ 1,
70
+ 0,
71
+ 250,
72
+ 0,
73
+ 2,
74
+ 4,
75
+ 0
76
+ ],
77
+ [
78
+ 307,
79
+ 13,
80
+ 6,
81
+ 8,
82
+ 29,
83
+ 66,
84
+ 30,
85
+ 32,
86
+ 9,
87
+ 0
88
+ ],
89
+ [
90
+ 0,
91
+ 1,
92
+ 1,
93
+ 0,
94
+ 4,
95
+ 1,
96
+ 3,
97
+ 590,
98
+ 0,
99
+ 0
100
+ ],
101
+ [
102
+ 22,
103
+ 3,
104
+ 1,
105
+ 27,
106
+ 17,
107
+ 13,
108
+ 0,
109
+ 4,
110
+ 406,
111
+ 7
112
+ ],
113
+ [
114
+ 6,
115
+ 10,
116
+ 107,
117
+ 0,
118
+ 3,
119
+ 4,
120
+ 0,
121
+ 0,
122
+ 10,
123
+ 460
124
+ ]
125
+ ],
126
+ "duration_seconds": 33.647279500961304,
127
+ "evaluation_identity": "13c57cd97dfa4bbb8efaecefc576a7c655e496fd8983f8d5f6f7efd3fbdc272d",
128
+ "image_manifest_hash": "d00f5f6e1e711aada1d545d6d1aee03cdc9d5a9e0c6431ec4ce6551002a3d17f",
129
+ "macro_f1": 0.5767701775459999,
130
+ "macro_precision": 0.5966315282484262,
131
+ "macro_recall": 0.6218333333333333,
132
+ "peak_gpu_memory_gib": 13.199361324310303,
133
+ "per_class": {
134
+ "accuracy": [
135
+ 0.795,
136
+ 0.9066666666666666,
137
+ 0.03166666666666667,
138
+ 0.572,
139
+ 0.666,
140
+ 0.625,
141
+ 0.06,
142
+ 0.9833333333333333,
143
+ 0.812,
144
+ 0.7666666666666667
145
+ ],
146
+ "f1": [
147
+ 0.6135048231511254,
148
+ 0.7640449438202247,
149
+ 0.048284625158831,
150
+ 0.639821029082774,
151
+ 0.6809815950920245,
152
+ 0.513874614594039,
153
+ 0.1079136690647482,
154
+ 0.8183079056865464,
155
+ 0.7415525114155251,
156
+ 0.8394160583941606
157
+ ],
158
+ "precision": [
159
+ 0.49947643979057593,
160
+ 0.6601941747572816,
161
+ 0.10160427807486631,
162
+ 0.7258883248730964,
163
+ 0.696652719665272,
164
+ 0.4363001745200698,
165
+ 0.5357142857142857,
166
+ 0.7007125890736342,
167
+ 0.6823529411764706,
168
+ 0.9274193548387096
169
+ ],
170
+ "recall": [
171
+ 0.795,
172
+ 0.9066666666666666,
173
+ 0.03166666666666667,
174
+ 0.572,
175
+ 0.666,
176
+ 0.625,
177
+ 0.06,
178
+ 0.9833333333333333,
179
+ 0.812,
180
+ 0.7666666666666667
181
+ ],
182
+ "support": [
183
+ 600,
184
+ 600,
185
+ 600,
186
+ 500,
187
+ 500,
188
+ 400,
189
+ 500,
190
+ 600,
191
+ 500,
192
+ 600
193
+ ]
194
+ },
195
+ "sample_count": 5400,
196
+ "timestamp_utc": "2026-11-05T02:34:15Z"
197
+ }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/predictions.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/progress.json CHANGED
@@ -1,12 +1,12 @@
1
  {
2
  "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
3
- "completed_batches": 40,
4
- "eta_seconds": 1.7384650204330683,
5
- "evaluated": 5120,
6
- "latest_chunk": "validation/epoch_002/chunks/00040.npz",
7
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
8
  "samples": 5400,
9
- "throughput": 161.06162431168693,
10
  "total_batches": 43,
11
- "updated_utc": "2026-11-05T02:34:13Z"
12
  }
 
1
  {
2
  "checkpoint_identity": "8da5676ea307fe7e460eb5f6578ce0201f3355d5681c135cc86859eb7796e0a4",
3
+ "completed_batches": 43,
4
+ "eta_seconds": 0.0,
5
+ "evaluated": 5400,
6
+ "latest_chunk": "validation/epoch_002/chunks/00043.npz",
7
  "metrics_status": "PROVISIONAL_SUBSET_NOT_FINAL",
8
  "samples": 5400,
9
+ "throughput": 160.72345090721953,
10
  "total_batches": 43,
11
+ "updated_utc": "2026-11-05T02:34:15Z"
12
  }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_002/recovery_journal.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "duration_basis": "measured optimizer steps + evaluator; excludes checkpoint serialization",
3
+ "epoch_duration_seconds": 47.328710079193115,
4
+ "validation_seconds": 35.53907918930054
5
+ }
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00001.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b4effea2f1682bfcadc86f2d36d1eb998f692d5348adc4fa545b41d34a74ad8
3
+ size 7732
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00002.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bdb765d40472253d1a4d8e6a57af47b18ae41655672582d87712355a33ba2146
3
+ size 7746
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00003.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5191510ec3d246bf6bed5ed1ec37dbc12f7be0abee642db8a27b328354ae9f6
3
+ size 7748
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00004.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af65422afaa074470034cc706bca690b53a284552784f5b8583ba3f434355ce1
3
+ size 7746
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00005.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3c0c1d9d734ebd66c42831420bfc8e6941cfd113169cc5df00a30f68b3c6c8d
3
+ size 7721
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00006.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a47da9363fa2acd9e154a9ec85265e770f6c2ff32130b2d4c112150edf74dca0
3
+ size 7744
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00007.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c6ff65fb517fd248d5d177f06a5f591bd1b66b44eb6d7641d811b6b8565f323
3
+ size 7725
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00008.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e3414030f95ec67254688c5fc69ddeca9f2d62f589a410d7b2d0fca34f5e1ff9
3
+ size 7754
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00009.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc9c858709c8a36acb5c0ca74511e89ab6658c0b7e7ace327d90c7dbd2b42389
3
+ size 7742
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00010.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1c4ba9ab1bb76aecee16f0a75bdb432b9bb08a533fc76ebb4a45d0af99ad268
3
+ size 7766
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00011.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2519ef1086f94c694dc84b5ef57ceb7ac71bf70930b7f6e032d414543f0144e5
3
+ size 7740
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00012.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e45aa94f4b1f11e241c27f51d3b6bd183541cb2c0917f0c92947afa05292c028
3
+ size 7763
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00013.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5117bf312ab6eb8aaeb07fa1a87dd1ae198b1de1775f57a9387c25c8edafb275
3
+ size 7741
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00014.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10a63aa5e8c111d7362222bfd3b4496100921a1963a7b194910d9bc8cc63736a
3
+ size 7753
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00015.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:563f0ea80418abb5c3d85e03b1ead50f131dfeb50975f51f3f4b488a40e014a7
3
+ size 7719
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00016.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4383e836a5e69637eaadb584a06a65f8aed9ad19f86314c0633d6bb83e430376
3
+ size 7718
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00017.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b754f471b0f041bfc751a086fbeec4c093e4fe6c28d8f315d409750d30c2d8f2
3
+ size 7785
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00018.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6d52c2ef2168bad63918597fb1bca3edeed0484307e4f40bff4de61223c73fe
3
+ size 7734
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00019.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc8ded5983018c26d3b8c0d25050c52faba1014384bf129fabf2a0b5f0576794
3
+ size 7745
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00020.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ac3d30ee7a564e036e4b4b82afc7275cd253550d542504e3a76836750d31f06
3
+ size 7762
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00021.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7505bf92cf9f94775e3f83b92c7462141e0180fa576bec611b64ce6b4d20f55b
3
+ size 7759
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00022.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fcedbf386537dd8c9b77f4771e063ecd81fd4c8f0d57a81d414aeffc028e5ce
3
+ size 7740
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00023.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49829107b6e97edd38f5bac1b2677a0905beae2d507230fb3ccee5ccc03e0a46
3
+ size 7772
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00024.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93a110e4d2df522b0d8abdde6c75835e23d88eae88f448093f7841be829dd619
3
+ size 7743
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00025.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5419499a9f4d55dfbafd30a568b5ad54cb8dbeb8814ad556007d7ba76854b423
3
+ size 7759
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00026.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89c371890dae84a0024a9617067dcfcb8a54b5c499596fcd30398564da61b01d
3
+ size 7757
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00027.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b11be05110251b5be4ec49ad8246052e535c18ed5d7481755a158be67ec2f8b3
3
+ size 7737
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00028.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cc3ed5170b8a0c1ce89af7a05de530aec207352c95bb11e55da4dc3c62150e2c
3
+ size 7748
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00029.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54d1eca28c69c4226e7d2f4dd000ec2958e0ec5f3304cba71e95cd4575aced88
3
+ size 7742
experiments/eurosat_spc_clip_20261104T054309Z/two_additional_45e/d45_e45_20261104T122347Z/experiment_E45/validation/epoch_003/chunks/00030.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77063c60809cfb096e9d3ca12776ee58ccfb16a707ab910d6a1d05aca3172c4c
3
+ size 7749