Epoch 12/40 snapshot

runs/cars_issynth_20261025_seed0_support1/checkpoints/current/latest.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e2f7bca27aca7ed66236a6e4a2ff9bc4515d8aa92a47f232fe47d200caf68101
-size 67609675
+oid sha256:4d61107d23fd409da76979a874a3bceafe2e19ae6bd59a27f55d28d44ffd37ea
+size 67609803

runs/cars_issynth_20261025_seed0_support1/logs/history.json

@@ -141,5 +141,18 @@
     "validation_loss": 0.24244166090292668,
     "validation_macro_f1": 0.9239497662495237,
     "validation_weighted_f1": 0.9251801536798633
+  },
+  {
+    "epoch": 11,
+    "epoch_seconds": 1125.0348768234253,
+    "global_step": 588,
+    "gpu_peak_allocated_gib": 9.631484508514404,
+    "learning_rate": 0.0001,
+    "timestamp_utc": "2026-10-25T09:05:36.956708+00:00",
+    "train_loss": 0.6737350541718152,
+    "validation_accuracy": 0.9258123850398529,
+    "validation_loss": 0.2395989412000697,
+    "validation_macro_f1": 0.9231459261881648,
+    "validation_weighted_f1": 0.9242427035185231
   }
 ]

runs/cars_issynth_20261025_seed0_support1/logs/metrics.csv

@@ -10,3 +10,4 @@ epoch,global_step,train_loss,validation_loss,validation_accuracy,validation_macr
 8,441,0.7753355259798012,0.260824340371498,0.9319435928877989,0.9282285435112299,0.9292310762433401,0.0001,1158.2063155174255,9.630891799926758,2026-10-25T07:39:43.853366+00:00
 9,490,0.7383950836804448,0.2573974011145411,0.9251992642550583,0.9213461713943657,0.9230517667048938,0.0001,1118.5864386558533,9.603536605834961,2026-10-25T08:27:45.992469+00:00
 10,539,0.7137451171875001,0.24244166090292668,0.9282648681790313,0.9239497662495237,0.9251801536798633,0.0001,1126.608634710312,9.743394374847412,2026-10-25T08:46:43.015992+00:00
+11,588,0.6737350541718152,0.2395989412000697,0.9258123850398529,0.9231459261881648,0.9242427035185231,0.0001,1125.0348768234253,9.631484508514404,2026-10-25T09:05:36.956708+00:00

runs/cars_issynth_20261025_seed0_support1/logs/status.json

@@ -1,20 +1,20 @@
 {
   "best_epoch": 7,
   "best_validation_accuracy": 0.9325567136725935,
-  "current_epoch": 11,
-  "elapsed_seconds": 12479.955367565155,
+  "current_epoch": 12,
+  "elapsed_seconds": 13604.99024438858,
   "error": null,
-  "estimated_remaining_seconds": 32901.70051448995,
-  "global_step": 539,
+  "estimated_remaining_seconds": 31744.97723690669,
+  "global_step": 588,
   "gpu_memory_allocated_gib": 4.065773963928223,
   "gpu_memory_reserved_gib": 14.94140625,
-  "latest_checkpoint_timestamp": 1792918003.138041,
-  "latest_hf_sync_timestamp": "2026-10-25T08:46:43.437162+00:00",
-  "latest_train_loss": 0.7137451171875001,
-  "latest_validation_accuracy": 0.9282648681790313,
-  "latest_validation_loss": 0.24244166090292668,
+  "latest_checkpoint_timestamp": 1792919137.078645,
+  "latest_hf_sync_timestamp": "2026-10-25T09:05:37.362569+00:00",
+  "latest_train_loss": 0.6737350541718152,
+  "latest_validation_accuracy": 0.9258123850398529,
+  "latest_validation_loss": 0.2395989412000697,
   "run_id": "cars_issynth_20261025_seed0_support1",
   "status": "RUNNING",
   "total_epochs": 40,
-  "updated_at_utc": "2026-10-25T08:46:43.438023+00:00"
+  "updated_at_utc": "2026-10-25T09:05:37.363461+00:00"
 }

runs/cars_issynth_20261025_seed0_support1/logs/supervisor.log

@@ -537,3 +537,259 @@ I0000 00:00:1792915733.588368    2431 port.cc:153] oneDNN custom operations are
 
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
 
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
 
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+[2026-10-25T08:46:45.826498+00:00] epoch=10 hf_snapshot_synced=true
+[2026-10-25T08:47:08.238689+00:00] step=540 batch=1/49 loss=0.748854 real=0.136826 synth=3.196968 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:47:26.537990+00:00] step=541 batch=2/49 loss=0.702161 real=0.215136 synth=2.650265 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:47:47.386757+00:00] step=542 batch=3/49 loss=0.651152 real=0.138884 synth=2.700226 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:09.318472+00:00] step=543 batch=4/49 loss=0.654296 real=0.113837 synth=2.816132 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:30.162283+00:00] step=544 batch=5/49 loss=0.672266 real=0.157249 synth=2.732330 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:50.936621+00:00] step=545 batch=6/49 loss=0.712038 real=0.159274 synth=2.923096 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:12.094487+00:00] step=546 batch=7/49 loss=0.691810 real=0.181401 synth=2.733444 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:34.980044+00:00] step=547 batch=8/49 loss=0.670874 real=0.125252 synth=2.853363 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:55.567060+00:00] step=548 batch=9/49 loss=0.622914 real=0.125710 synth=2.611732 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:16.401282+00:00] step=549 batch=10/49 loss=0.647417 real=0.125139 synth=2.736526 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:36.219194+00:00] step=550 batch=11/49 loss=0.721012 real=0.147763 synth=3.014008 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:56.685383+00:00] step=551 batch=12/49 loss=0.653756 real=0.129031 synth=2.752655 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:51:20.421009+00:00] step=552 batch=13/49 loss=0.690850 real=0.112761 synth=3.003204 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:51:39.839677+00:00] step=553 batch=14/49 loss=0.599432 real=0.137321 synth=2.447876 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:00.804080+00:00] step=554 batch=15/49 loss=0.697163 real=0.178518 synth=2.771744 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:22.162292+00:00] step=555 batch=16/49 loss=0.643278 real=0.170007 synth=2.536362 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:43.722662+00:00] step=556 batch=17/49 loss=0.624819 real=0.141982 synth=2.556168 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:03.758913+00:00] step=557 batch=18/49 loss=0.671659 real=0.162657 synth=2.707668 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:26.272368+00:00] step=558 batch=19/49 loss=0.737734 real=0.126276 synth=3.183563 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:44.641162+00:00] step=559 batch=20/49 loss=0.741034 real=0.131714 synth=3.178314 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:04.397711+00:00] step=560 batch=21/49 loss=0.764481 real=0.181455 synth=3.096588 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:22.613855+00:00] step=561 batch=22/49 loss=0.726380 real=0.185304 synth=2.890686 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:42.064510+00:00] step=562 batch=23/49 loss=0.617532 real=0.142680 synth=2.516937 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:03.987121+00:00] step=563 batch=24/49 loss=0.646523 real=0.148134 synth=2.640076 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:24.346030+00:00] step=564 batch=25/49 loss=0.704816 real=0.246179 synth=2.539368 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:43.550408+00:00] step=565 batch=26/49 loss=0.650599 real=0.145730 synth=2.670074 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:03.735901+00:00] step=566 batch=27/49 loss=0.740027 real=0.147141 synth=3.111572 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:28.061382+00:00] step=567 batch=28/49 loss=0.645296 real=0.141333 synth=2.661148 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:48.560381+00:00] step=568 batch=29/49 loss=0.698376 real=0.188121 synth=2.739395 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:09.630724+00:00] step=569 batch=30/49 loss=0.717010 real=0.122367 synth=3.095581 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:28.133662+00:00] step=570 batch=31/49 loss=0.626909 real=0.128386 synth=2.621002 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:48.452273+00:00] step=571 batch=32/49 loss=0.595103 real=0.143215 synth=2.402657 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:10.826087+00:00] step=572 batch=33/49 loss=0.590667 real=0.146842 synth=2.365967 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:30.932548+00:00] step=573 batch=34/49 loss=0.701126 real=0.142453 synth=2.935822 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:52.681013+00:00] step=574 batch=35/49 loss=0.645982 real=0.150984 synth=2.625977 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:15.749173+00:00] step=575 batch=36/49 loss=0.588168 real=0.127273 synth=2.431747 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:36.952527+00:00] step=576 batch=37/49 loss=0.654770 real=0.181742 synth=2.546883 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:58.205574+00:00] step=577 batch=38/49 loss=0.673269 real=0.150372 synth=2.764854 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:17.688823+00:00] step=578 batch=39/49 loss=0.685816 real=0.157046 synth=2.800896 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:40.139353+00:00] step=579 batch=40/49 loss=0.630363 real=0.131623 synth=2.625320 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:57.229632+00:00] step=580 batch=41/49 loss=0.661896 real=0.210217 synth=2.468613 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:16.656492+00:00] step=581 batch=42/49 loss=0.685889 real=0.141129 synth=2.864929 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:38.597825+00:00] step=582 batch=43/49 loss=0.687672 real=0.152124 synth=2.829865 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:59.258206+00:00] step=583 batch=44/49 loss=0.677473 real=0.137309 synth=2.838127 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:02:20.327901+00:00] step=584 batch=45/49 loss=0.622527 real=0.151465 synth=2.506775 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:02:42.082668+00:00] step=585 batch=46/49 loss=0.741032 real=0.181062 synth=2.980911 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:01.993828+00:00] step=586 batch=47/49 loss=0.636971 real=0.143143 synth=2.612286 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:23.745453+00:00] step=587 batch=48/49 loss=0.698167 real=0.133894 synth=2.955261 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:41.841284+00:00] step=588 batch=49/49 loss=0.743658 real=0.171024 synth=3.034195 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:05:37.362679+00:00] epoch=11 val_accuracy=0.925812 val_macro_f1=0.923146 best=0.932557 sync=pending
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:  16%|█▋        | 11.1MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:  25%|██▍       | 16.7MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:  35%|███▌      | 24.0MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:  60%|██████    | 40.8MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt:  81%|████████  | 54.7MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
+
+
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            

runs/cars_issynth_20261025_seed0_support1/logs/train.log

@@ -560,3 +560,54 @@
 [2026-10-25T08:44:28.824644+00:00] step=538 batch=48/49 loss=0.676711 real=0.216715 synth=2.516693 lr=0.0001 gpu_alloc_gib=4.085
 [2026-10-25T08:44:49.829752+00:00] step=539 batch=49/49 loss=0.682344 real=0.120941 synth=2.927956 lr=0.0001 gpu_alloc_gib=4.085
 [2026-10-25T08:46:43.437273+00:00] epoch=10 val_accuracy=0.928265 val_macro_f1=0.923950 best=0.932557 sync=pending
+[2026-10-25T08:46:45.826498+00:00] epoch=10 hf_snapshot_synced=true
+[2026-10-25T08:47:08.238689+00:00] step=540 batch=1/49 loss=0.748854 real=0.136826 synth=3.196968 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:47:26.537990+00:00] step=541 batch=2/49 loss=0.702161 real=0.215136 synth=2.650265 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:47:47.386757+00:00] step=542 batch=3/49 loss=0.651152 real=0.138884 synth=2.700226 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:09.318472+00:00] step=543 batch=4/49 loss=0.654296 real=0.113837 synth=2.816132 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:30.162283+00:00] step=544 batch=5/49 loss=0.672266 real=0.157249 synth=2.732330 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:48:50.936621+00:00] step=545 batch=6/49 loss=0.712038 real=0.159274 synth=2.923096 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:12.094487+00:00] step=546 batch=7/49 loss=0.691810 real=0.181401 synth=2.733444 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:34.980044+00:00] step=547 batch=8/49 loss=0.670874 real=0.125252 synth=2.853363 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:49:55.567060+00:00] step=548 batch=9/49 loss=0.622914 real=0.125710 synth=2.611732 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:16.401282+00:00] step=549 batch=10/49 loss=0.647417 real=0.125139 synth=2.736526 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:36.219194+00:00] step=550 batch=11/49 loss=0.721012 real=0.147763 synth=3.014008 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:50:56.685383+00:00] step=551 batch=12/49 loss=0.653756 real=0.129031 synth=2.752655 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:51:20.421009+00:00] step=552 batch=13/49 loss=0.690850 real=0.112761 synth=3.003204 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:51:39.839677+00:00] step=553 batch=14/49 loss=0.599432 real=0.137321 synth=2.447876 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:00.804080+00:00] step=554 batch=15/49 loss=0.697163 real=0.178518 synth=2.771744 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:22.162292+00:00] step=555 batch=16/49 loss=0.643278 real=0.170007 synth=2.536362 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:52:43.722662+00:00] step=556 batch=17/49 loss=0.624819 real=0.141982 synth=2.556168 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:03.758913+00:00] step=557 batch=18/49 loss=0.671659 real=0.162657 synth=2.707668 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:26.272368+00:00] step=558 batch=19/49 loss=0.737734 real=0.126276 synth=3.183563 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:53:44.641162+00:00] step=559 batch=20/49 loss=0.741034 real=0.131714 synth=3.178314 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:04.397711+00:00] step=560 batch=21/49 loss=0.764481 real=0.181455 synth=3.096588 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:22.613855+00:00] step=561 batch=22/49 loss=0.726380 real=0.185304 synth=2.890686 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:54:42.064510+00:00] step=562 batch=23/49 loss=0.617532 real=0.142680 synth=2.516937 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:03.987121+00:00] step=563 batch=24/49 loss=0.646523 real=0.148134 synth=2.640076 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:24.346030+00:00] step=564 batch=25/49 loss=0.704816 real=0.246179 synth=2.539368 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:55:43.550408+00:00] step=565 batch=26/49 loss=0.650599 real=0.145730 synth=2.670074 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:03.735901+00:00] step=566 batch=27/49 loss=0.740027 real=0.147141 synth=3.111572 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:28.061382+00:00] step=567 batch=28/49 loss=0.645296 real=0.141333 synth=2.661148 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:56:48.560381+00:00] step=568 batch=29/49 loss=0.698376 real=0.188121 synth=2.739395 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:09.630724+00:00] step=569 batch=30/49 loss=0.717010 real=0.122367 synth=3.095581 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:28.133662+00:00] step=570 batch=31/49 loss=0.626909 real=0.128386 synth=2.621002 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:57:48.452273+00:00] step=571 batch=32/49 loss=0.595103 real=0.143215 synth=2.402657 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:10.826087+00:00] step=572 batch=33/49 loss=0.590667 real=0.146842 synth=2.365967 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:30.932548+00:00] step=573 batch=34/49 loss=0.701126 real=0.142453 synth=2.935822 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:58:52.681013+00:00] step=574 batch=35/49 loss=0.645982 real=0.150984 synth=2.625977 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:15.749173+00:00] step=575 batch=36/49 loss=0.588168 real=0.127273 synth=2.431747 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:36.952527+00:00] step=576 batch=37/49 loss=0.654770 real=0.181742 synth=2.546883 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T08:59:58.205574+00:00] step=577 batch=38/49 loss=0.673269 real=0.150372 synth=2.764854 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:17.688823+00:00] step=578 batch=39/49 loss=0.685816 real=0.157046 synth=2.800896 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:40.139353+00:00] step=579 batch=40/49 loss=0.630363 real=0.131623 synth=2.625320 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:00:57.229632+00:00] step=580 batch=41/49 loss=0.661896 real=0.210217 synth=2.468613 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:16.656492+00:00] step=581 batch=42/49 loss=0.685889 real=0.141129 synth=2.864929 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:38.597825+00:00] step=582 batch=43/49 loss=0.687672 real=0.152124 synth=2.829865 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:01:59.258206+00:00] step=583 batch=44/49 loss=0.677473 real=0.137309 synth=2.838127 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:02:20.327901+00:00] step=584 batch=45/49 loss=0.622527 real=0.151465 synth=2.506775 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:02:42.082668+00:00] step=585 batch=46/49 loss=0.741032 real=0.181062 synth=2.980911 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:01.993828+00:00] step=586 batch=47/49 loss=0.636971 real=0.143143 synth=2.612286 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:23.745453+00:00] step=587 batch=48/49 loss=0.698167 real=0.133894 synth=2.955261 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:03:41.841284+00:00] step=588 batch=49/49 loss=0.743658 real=0.171024 synth=3.034195 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T09:05:37.362679+00:00] epoch=11 val_accuracy=0.925812 val_macro_f1=0.923146 best=0.932557 sync=pending

runs/cars_issynth_20261025_seed0_support1/metrics/gpu.csv

@@ -75,3 +75,41 @@ timestamp_utc,gpu_util_percent,memory_used_mib,memory_free_mib,temperature_c,pow
 2026-10-25T08:45:17.883243+00:00,16,16023,81230,43,288.52
 2026-10-25T08:45:47.922271+00:00,100,16023,81230,49,322.22
 2026-10-25T08:46:17.968457+00:00,98,16023,81230,48,281.42
+2026-10-25T08:46:48.008534+00:00,7,16023,81230,39,88.32
+2026-10-25T08:47:18.045554+00:00,71,16023,81230,43,316.49
+2026-10-25T08:47:48.092498+00:00,2,16023,81230,42,228.60
+2026-10-25T08:48:18.133561+00:00,100,16023,81230,46,344.45
+2026-10-25T08:48:48.204903+00:00,96,16023,81230,43,307.04
+2026-10-25T08:49:18.246473+00:00,62,16023,81230,43,253.16
+2026-10-25T08:49:48.286943+00:00,0,16023,81230,48,316.07
+2026-10-25T08:50:18.328671+00:00,60,16023,81230,45,245.79
+2026-10-25T08:50:48.375121+00:00,80,16023,81230,45,334.53
+2026-10-25T08:51:18.416725+00:00,52,16023,81230,47,321.79
+2026-10-25T08:51:48.459914+00:00,20,16023,81230,47,297.58
+2026-10-25T08:52:18.508505+00:00,100,16023,81230,48,363.35
+2026-10-25T08:52:48.554854+00:00,66,16023,81230,45,251.03
+2026-10-25T08:53:18.597149+00:00,99,16023,81230,51,352.11
+2026-10-25T08:53:48.638234+00:00,63,16023,81230,45,263.78
+2026-10-25T08:54:18.675554+00:00,100,16023,81230,48,355.82
+2026-10-25T08:54:48.723690+00:00,62,16023,81230,45,252.89
+2026-10-25T08:55:18.764526+00:00,99,16023,81230,50,357.30
+2026-10-25T08:55:48.805389+00:00,62,16023,81230,44,254.13
+2026-10-25T08:56:18.848647+00:00,100,16023,81230,48,348.52
+2026-10-25T08:56:48.898824+00:00,59,16023,81230,44,290.27
+2026-10-25T08:57:18.943224+00:00,100,16023,81230,47,343.72
+2026-10-25T08:57:48.987677+00:00,2,16023,81230,42,267.57
+2026-10-25T08:58:19.034053+00:00,100,16023,81230,47,283.04
+2026-10-25T08:58:49.076158+00:00,100,16023,81230,47,363.83
+2026-10-25T08:59:19.121811+00:00,61,16023,81230,45,263.20
+2026-10-25T08:59:49.169968+00:00,100,16023,81230,46,349.87
+2026-10-25T09:00:19.213657+00:00,62,16023,81230,46,222.50
+2026-10-25T09:00:49.253815+00:00,97,16023,81230,48,346.36
+2026-10-25T09:01:19.292302+00:00,70,16023,81230,46,259.33
+2026-10-25T09:01:49.332192+00:00,100,16023,81230,47,341.27
+2026-10-25T09:02:19.375479+00:00,38,16023,81230,45,310.74
+2026-10-25T09:02:49.412463+00:00,59,16023,81230,45,261.61
+2026-10-25T09:03:19.452220+00:00,92,16023,81230,50,352.38
+2026-10-25T09:03:49.498998+00:00,0,16023,81230,39,81.75
+2026-10-25T09:04:19.530807+00:00,0,16023,81230,42,378.52
+2026-10-25T09:04:49.572968+00:00,100,16023,81230,51,260.70
+2026-10-25T09:05:19.620709+00:00,54,16023,81230,47,237.91