Epoch 29/40 snapshot

runs/cars_issynth_20261025_seed0_support1/checkpoints/current/latest.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddd01ef47e1849d0406e72622c41c841bd50a6e18653a6d13c5a633b6851ffb5
-size 67612619
+oid sha256:87b2ecbec284ccdcb97a3480b7d672552301ac4efc62f5f0b5eff54cafde5af7
+size 67612747

runs/cars_issynth_20261025_seed0_support1/logs/history.json

@@ -362,5 +362,18 @@
     "validation_loss": 0.20661631668069463,
     "validation_macro_f1": 0.9301118674856591,
     "validation_weighted_f1": 0.9318980096278491
+  },
+  {
+    "epoch": 28,
+    "epoch_seconds": 1156.264847278595,
+    "global_step": 1421,
+    "gpu_peak_allocated_gib": 9.668413162231445,
+    "learning_rate": 0.0001,
+    "timestamp_utc": "2026-10-25T14:34:03.918310+00:00",
+    "train_loss": 0.360425158909389,
+    "validation_accuracy": 0.923973022685469,
+    "validation_loss": 0.22071337118416812,
+    "validation_macro_f1": 0.9209038557010585,
+    "validation_weighted_f1": 0.9223615603044708
   }
 ]

runs/cars_issynth_20261025_seed0_support1/logs/metrics.csv

@@ -27,3 +27,4 @@ epoch,global_step,train_loss,validation_loss,validation_accuracy,validation_macr
 25,1274,0.4029264138669384,0.21226682162274293,0.927038626609442,0.9243063502544927,0.9262645001176035,0.0001,1160.5145826339722,9.969139099121094,2026-10-25T13:35:58.799441+00:00
 26,1323,0.383632823399135,0.21702664971525068,0.9307173513182097,0.9270378633778205,0.9290955093346359,0.0001,1143.33811378479,9.841930866241455,2026-10-25T13:55:13.426459+00:00
 27,1372,0.3694936752319336,0.20661631668069463,0.9331698344573881,0.9301118674856591,0.9318980096278491,0.0001,1155.6774883270264,9.862995624542236,2026-10-25T14:14:38.035257+00:00
+28,1421,0.360425158909389,0.22071337118416812,0.923973022685469,0.9209038557010585,0.9223615603044708,0.0001,1156.264847278595,9.668413162231445,2026-10-25T14:34:03.918310+00:00

runs/cars_issynth_20261025_seed0_support1/logs/status.json

@@ -1,20 +1,20 @@
 {
   "best_epoch": 27,
   "best_validation_accuracy": 0.9331698344573881,
-  "current_epoch": 28,
-  "elapsed_seconds": 31996.242683172226,
+  "current_epoch": 29,
+  "elapsed_seconds": 33152.50753045082,
   "error": null,
-  "estimated_remaining_seconds": 13712.67543564524,
-  "global_step": 1372,
+  "estimated_remaining_seconds": 12575.089063274449,
+  "global_step": 1421,
   "gpu_memory_allocated_gib": 4.065773963928223,
   "gpu_memory_reserved_gib": 14.94140625,
-  "latest_checkpoint_timestamp": 1792937678.1603882,
-  "latest_hf_sync_timestamp": "2026-10-25T14:14:38.558844+00:00",
-  "latest_train_loss": 0.3694936752319336,
-  "latest_validation_accuracy": 0.9331698344573881,
-  "latest_validation_loss": 0.20661631668069463,
+  "latest_checkpoint_timestamp": 1792938844.0376244,
+  "latest_hf_sync_timestamp": "2026-10-25T14:34:04.351510+00:00",
+  "latest_train_loss": 0.360425158909389,
+  "latest_validation_accuracy": 0.923973022685469,
+  "latest_validation_loss": 0.22071337118416812,
   "run_id": "cars_issynth_20261025_seed0_support1",
   "status": "RUNNING",
   "total_epochs": 40,
-  "updated_at_utc": "2026-10-25T14:14:38.559673+00:00"
+  "updated_at_utc": "2026-10-25T14:34:04.352417+00:00"
 }

runs/cars_issynth_20261025_seed0_support1/logs/supervisor.log

@@ -7479,3 +7479,506 @@ I0000 00:00:1792915733.588368    2431 port.cc:153] oneDNN custom operations are
 
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
 
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
 
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+[2026-10-25T14:14:41.152585+00:00] epoch=27 hf_snapshot_synced=true
+[2026-10-25T14:15:05.840880+00:00] step=1373 batch=1/49 loss=0.366725 real=0.091439 synth=1.467869 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:15:25.465207+00:00] step=1374 batch=2/49 loss=0.367197 real=0.062428 synth=1.586273 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:15:45.174324+00:00] step=1375 batch=3/49 loss=0.298701 real=0.051124 synth=1.289009 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:07.996955+00:00] step=1376 batch=4/49 loss=0.401286 real=0.065617 synth=1.743965 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:29.680095+00:00] step=1377 batch=5/49 loss=0.401040 real=0.057304 synth=1.775986 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:52.809111+00:00] step=1378 batch=6/49 loss=0.340315 real=0.041560 synth=1.535336 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:13.663604+00:00] step=1379 batch=7/49 loss=0.290183 real=0.055545 synth=1.228735 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:34.683070+00:00] step=1380 batch=8/49 loss=0.370541 real=0.053551 synth=1.638504 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:56.881824+00:00] step=1381 batch=9/49 loss=0.361449 real=0.047879 synth=1.615730 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:18:20.665330+00:00] step=1382 batch=10/49 loss=0.335406 real=0.050264 synth=1.475975 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:18:41.872446+00:00] step=1383 batch=11/49 loss=0.382479 real=0.043457 synth=1.738569 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:05.320473+00:00] step=1384 batch=12/49 loss=0.369510 real=0.052783 synth=1.636417 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:23.384433+00:00] step=1385 batch=13/49 loss=0.351548 real=0.058040 synth=1.525581 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:42.432250+00:00] step=1386 batch=14/49 loss=0.339041 real=0.065685 synth=1.432465 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:02.978193+00:00] step=1387 batch=15/49 loss=0.343696 real=0.047195 synth=1.529697 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:24.204633+00:00] step=1388 batch=16/49 loss=0.349130 real=0.058194 synth=1.512875 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:45.720201+00:00] step=1389 batch=17/49 loss=0.283486 real=0.050677 synth=1.214720 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:08.905153+00:00] step=1390 batch=18/49 loss=0.385844 real=0.059359 synth=1.691788 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:28.382020+00:00] step=1391 batch=19/49 loss=0.286804 real=0.062700 synth=1.183218 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:48.674376+00:00] step=1392 batch=20/49 loss=0.304076 real=0.056815 synth=1.293119 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:11.729268+00:00] step=1393 batch=21/49 loss=0.374902 real=0.052071 synth=1.666227 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:33.398153+00:00] step=1394 batch=22/49 loss=0.386029 real=0.042963 synth=1.758293 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:56.583649+00:00] step=1395 batch=23/49 loss=0.351320 real=0.045776 synth=1.573494 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:17.050342+00:00] step=1396 batch=24/49 loss=0.407875 real=0.051142 synth=1.834808 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:37.293978+00:00] step=1397 batch=25/49 loss=0.438808 real=0.048668 synth=1.999367 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:59.648037+00:00] step=1398 batch=26/49 loss=0.403540 real=0.053430 synth=1.803982 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:24:18.237395+00:00] step=1399 batch=27/49 loss=0.369341 real=0.052716 synth=1.635838 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:24:40.676836+00:00] step=1400 batch=28/49 loss=0.337870 real=0.049952 synth=1.489543 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:03.002870+00:00] step=1401 batch=29/49 loss=0.445731 real=0.061234 synth=1.983719 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:25.747060+00:00] step=1402 batch=30/49 loss=0.349345 real=0.052973 synth=1.534832 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:48.595658+00:00] step=1403 batch=31/49 loss=0.383395 real=0.053007 synth=1.704945 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:10.580695+00:00] step=1404 batch=32/49 loss=0.314602 real=0.051136 synth=1.368465 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:35.104385+00:00] step=1405 batch=33/49 loss=0.376224 real=0.056440 synth=1.655359 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:54.864167+00:00] step=1406 batch=34/49 loss=0.423378 real=0.071054 synth=1.832670 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:14.977976+00:00] step=1407 batch=35/49 loss=0.351336 real=0.064136 synth=1.500135 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:34.669106+00:00] step=1408 batch=36/49 loss=0.335198 real=0.048862 synth=1.480543 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:58.363447+00:00] step=1409 batch=37/49 loss=0.336406 real=0.059486 synth=1.444084 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:15.363965+00:00] step=1410 batch=38/49 loss=0.439182 real=0.073122 synth=1.903419 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:37.150963+00:00] step=1411 batch=39/49 loss=0.358735 real=0.073059 synth=1.501440 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:58.785358+00:00] step=1412 batch=40/49 loss=0.334595 real=0.067694 synth=1.402203 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:29:22.099183+00:00] step=1413 batch=41/49 loss=0.338869 real=0.044166 synth=1.517681 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:29:41.640720+00:00] step=1414 batch=42/49 loss=0.325225 real=0.064788 synth=1.366972 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:02.884271+00:00] step=1415 batch=43/49 loss=0.386018 real=0.048769 synth=1.735016 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:23.843935+00:00] step=1416 batch=44/49 loss=0.361469 real=0.055638 synth=1.584793 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:44.264788+00:00] step=1417 batch=45/49 loss=0.423376 real=0.062565 synth=1.866623 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:05.477335+00:00] step=1418 batch=46/49 loss=0.348693 real=0.042788 synth=1.572315 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:24.114470+00:00] step=1419 batch=47/49 loss=0.365514 real=0.049933 synth=1.627838 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:45.278983+00:00] step=1420 batch=48/49 loss=0.305619 real=0.047333 synth=1.338764 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:32:05.005017+00:00] step=1421 batch=49/49 loss=0.359778 real=0.052888 synth=1.587337 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:34:04.351632+00:00] epoch=28 val_accuracy=0.923973 val_macro_f1=0.920904 best=0.933170 sync=pending
+
+
+
+
  ...nts/periodic/epoch_014.pt:  99%|█████████▉| 67.2MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt:  99%|█████████▉| 67.2MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt:  99%|█████████▉| 67.2MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:   0%|          | 60.7kB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:  13%|█▎        | 8.99MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:  48%|████▊     | 32.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt:  92%|█████████▏| 62.5MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB            
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            
+
+
+
+
+
+
+
+
+
+
+
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB            
+
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB            

runs/cars_issynth_20261025_seed0_support1/logs/train.log

@@ -1427,3 +1427,54 @@
 [2026-10-25T14:12:17.779804+00:00] step=1371 batch=48/49 loss=0.366579 real=0.047874 synth=1.641397 lr=0.0001 gpu_alloc_gib=4.085
 [2026-10-25T14:12:39.828917+00:00] step=1372 batch=49/49 loss=0.408649 real=0.061543 synth=1.797077 lr=0.0001 gpu_alloc_gib=4.085
 [2026-10-25T14:14:38.558952+00:00] epoch=27 val_accuracy=0.933170 val_macro_f1=0.930112 best=0.933170 sync=pending
+[2026-10-25T14:14:41.152585+00:00] epoch=27 hf_snapshot_synced=true
+[2026-10-25T14:15:05.840880+00:00] step=1373 batch=1/49 loss=0.366725 real=0.091439 synth=1.467869 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:15:25.465207+00:00] step=1374 batch=2/49 loss=0.367197 real=0.062428 synth=1.586273 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:15:45.174324+00:00] step=1375 batch=3/49 loss=0.298701 real=0.051124 synth=1.289009 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:07.996955+00:00] step=1376 batch=4/49 loss=0.401286 real=0.065617 synth=1.743965 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:29.680095+00:00] step=1377 batch=5/49 loss=0.401040 real=0.057304 synth=1.775986 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:16:52.809111+00:00] step=1378 batch=6/49 loss=0.340315 real=0.041560 synth=1.535336 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:13.663604+00:00] step=1379 batch=7/49 loss=0.290183 real=0.055545 synth=1.228735 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:34.683070+00:00] step=1380 batch=8/49 loss=0.370541 real=0.053551 synth=1.638504 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:17:56.881824+00:00] step=1381 batch=9/49 loss=0.361449 real=0.047879 synth=1.615730 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:18:20.665330+00:00] step=1382 batch=10/49 loss=0.335406 real=0.050264 synth=1.475975 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:18:41.872446+00:00] step=1383 batch=11/49 loss=0.382479 real=0.043457 synth=1.738569 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:05.320473+00:00] step=1384 batch=12/49 loss=0.369510 real=0.052783 synth=1.636417 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:23.384433+00:00] step=1385 batch=13/49 loss=0.351548 real=0.058040 synth=1.525581 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:19:42.432250+00:00] step=1386 batch=14/49 loss=0.339041 real=0.065685 synth=1.432465 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:02.978193+00:00] step=1387 batch=15/49 loss=0.343696 real=0.047195 synth=1.529697 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:24.204633+00:00] step=1388 batch=16/49 loss=0.349130 real=0.058194 synth=1.512875 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:20:45.720201+00:00] step=1389 batch=17/49 loss=0.283486 real=0.050677 synth=1.214720 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:08.905153+00:00] step=1390 batch=18/49 loss=0.385844 real=0.059359 synth=1.691788 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:28.382020+00:00] step=1391 batch=19/49 loss=0.286804 real=0.062700 synth=1.183218 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:21:48.674376+00:00] step=1392 batch=20/49 loss=0.304076 real=0.056815 synth=1.293119 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:11.729268+00:00] step=1393 batch=21/49 loss=0.374902 real=0.052071 synth=1.666227 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:33.398153+00:00] step=1394 batch=22/49 loss=0.386029 real=0.042963 synth=1.758293 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:22:56.583649+00:00] step=1395 batch=23/49 loss=0.351320 real=0.045776 synth=1.573494 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:17.050342+00:00] step=1396 batch=24/49 loss=0.407875 real=0.051142 synth=1.834808 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:37.293978+00:00] step=1397 batch=25/49 loss=0.438808 real=0.048668 synth=1.999367 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:23:59.648037+00:00] step=1398 batch=26/49 loss=0.403540 real=0.053430 synth=1.803982 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:24:18.237395+00:00] step=1399 batch=27/49 loss=0.369341 real=0.052716 synth=1.635838 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:24:40.676836+00:00] step=1400 batch=28/49 loss=0.337870 real=0.049952 synth=1.489543 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:03.002870+00:00] step=1401 batch=29/49 loss=0.445731 real=0.061234 synth=1.983719 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:25.747060+00:00] step=1402 batch=30/49 loss=0.349345 real=0.052973 synth=1.534832 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:25:48.595658+00:00] step=1403 batch=31/49 loss=0.383395 real=0.053007 synth=1.704945 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:10.580695+00:00] step=1404 batch=32/49 loss=0.314602 real=0.051136 synth=1.368465 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:35.104385+00:00] step=1405 batch=33/49 loss=0.376224 real=0.056440 synth=1.655359 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:26:54.864167+00:00] step=1406 batch=34/49 loss=0.423378 real=0.071054 synth=1.832670 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:14.977976+00:00] step=1407 batch=35/49 loss=0.351336 real=0.064136 synth=1.500135 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:34.669106+00:00] step=1408 batch=36/49 loss=0.335198 real=0.048862 synth=1.480543 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:27:58.363447+00:00] step=1409 batch=37/49 loss=0.336406 real=0.059486 synth=1.444084 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:15.363965+00:00] step=1410 batch=38/49 loss=0.439182 real=0.073122 synth=1.903419 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:37.150963+00:00] step=1411 batch=39/49 loss=0.358735 real=0.073059 synth=1.501440 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:28:58.785358+00:00] step=1412 batch=40/49 loss=0.334595 real=0.067694 synth=1.402203 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:29:22.099183+00:00] step=1413 batch=41/49 loss=0.338869 real=0.044166 synth=1.517681 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:29:41.640720+00:00] step=1414 batch=42/49 loss=0.325225 real=0.064788 synth=1.366972 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:02.884271+00:00] step=1415 batch=43/49 loss=0.386018 real=0.048769 synth=1.735016 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:23.843935+00:00] step=1416 batch=44/49 loss=0.361469 real=0.055638 synth=1.584793 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:30:44.264788+00:00] step=1417 batch=45/49 loss=0.423376 real=0.062565 synth=1.866623 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:05.477335+00:00] step=1418 batch=46/49 loss=0.348693 real=0.042788 synth=1.572315 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:24.114470+00:00] step=1419 batch=47/49 loss=0.365514 real=0.049933 synth=1.627838 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:31:45.278983+00:00] step=1420 batch=48/49 loss=0.305619 real=0.047333 synth=1.338764 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:32:05.005017+00:00] step=1421 batch=49/49 loss=0.359778 real=0.052888 synth=1.587337 lr=0.0001 gpu_alloc_gib=4.085
+[2026-10-25T14:34:04.351632+00:00] epoch=28 val_accuracy=0.923973 val_macro_f1=0.920904 best=0.933170 sync=pending

runs/cars_issynth_20261025_seed0_support1/metrics/gpu.csv

@@ -730,3 +730,42 @@ timestamp_utc,gpu_util_percent,memory_used_mib,memory_free_mib,temperature_c,pow
 2026-10-25T14:13:16.721100+00:00,0,16023,81230,40,268.37
 2026-10-25T14:13:46.767250+00:00,94,16023,81230,48,292.94
 2026-10-25T14:14:16.815607+00:00,96,16023,81230,49,260.83
+2026-10-25T14:14:46.861210+00:00,57,16023,81230,43,275.94
+2026-10-25T14:15:16.904473+00:00,37,16023,81230,42,291.47
+2026-10-25T14:15:46.944845+00:00,61,16023,81230,45,266.18
+2026-10-25T14:16:17.027238+00:00,98,16023,81230,48,319.61
+2026-10-25T14:16:47.067349+00:00,72,16023,81230,47,271.61
+2026-10-25T14:17:17.110304+00:00,67,16023,81230,44,253.79
+2026-10-25T14:17:47.156562+00:00,100,16023,81230,46,321.04
+2026-10-25T14:18:17.201368+00:00,100,16023,81230,49,370.31
+2026-10-25T14:18:47.239852+00:00,62,16023,81230,44,245.79
+2026-10-25T14:19:17.283542+00:00,12,16023,81230,42,323.07
+2026-10-25T14:19:47.326003+00:00,66,16023,81230,45,237.99
+2026-10-25T14:20:17.365326+00:00,98,16023,81230,46,372.94
+2026-10-25T14:20:47.405622+00:00,66,16023,81230,46,260.57
+2026-10-25T14:21:17.454539+00:00,41,16023,81230,46,249.55
+2026-10-25T14:21:47.499313+00:00,79,16023,81230,48,287.07
+2026-10-25T14:22:17.542617+00:00,61,16023,81230,45,260.77
+2026-10-25T14:22:47.590121+00:00,100,16023,81230,48,346.93
+2026-10-25T14:23:17.632394+00:00,2,16023,81230,42,234.09
+2026-10-25T14:23:47.673612+00:00,94,16023,81230,49,328.18
+2026-10-25T14:24:17.715857+00:00,62,16023,81230,47,292.82
+2026-10-25T14:24:47.757270+00:00,63,16023,81230,45,241.32
+2026-10-25T14:25:17.803371+00:00,43,16023,81230,42,334.98
+2026-10-25T14:25:47.843898+00:00,43,16023,81230,48,306.05
+2026-10-25T14:26:17.890703+00:00,61,16023,81230,45,224.17
+2026-10-25T14:26:47.933009+00:00,97,16023,81230,47,301.45
+2026-10-25T14:27:17.970715+00:00,61,16023,81230,43,254.28
+2026-10-25T14:27:48.011382+00:00,100,16023,81230,46,378.80
+2026-10-25T14:28:18.054215+00:00,56,16023,81230,43,239.98
+2026-10-25T14:28:48.098715+00:00,76,16023,81230,46,325.86
+2026-10-25T14:29:18.146803+00:00,100,16023,81230,46,354.77
+2026-10-25T14:29:48.196996+00:00,66,16023,81230,45,251.67
+2026-10-25T14:30:18.234309+00:00,62,16023,81230,49,341.20
+2026-10-25T14:30:48.284236+00:00,62,16023,81230,45,258.87
+2026-10-25T14:31:18.326432+00:00,92,16023,81230,49,342.52
+2026-10-25T14:31:48.373972+00:00,68,16023,81230,46,258.30
+2026-10-25T14:32:18.416754+00:00,99,16023,81230,48,267.56
+2026-10-25T14:32:48.468161+00:00,0,16023,81230,40,248.43
+2026-10-25T14:33:18.513855+00:00,100,16023,81230,50,281.39
+2026-10-25T14:33:48.557343+00:00,50,16023,81230,48,324.55