DanielNguyen121 commited on
Commit
27bc78f
·
verified ·
1 Parent(s): 75c4416

Epoch 26/40 snapshot

Browse files
runs/cars_issynth_20261025_seed0_support1/checkpoints/current/latest.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:46e74e4e21c275cbd5f1493ce86c7957e9d9f69755db0a4c4f3e29b53ccd2b8a
3
- size 67612043
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:406e28b7a9daf880b8cdc4d6cb47d03912b6c8358fee2af2abcec06a7535bf53
3
+ size 67612235
runs/cars_issynth_20261025_seed0_support1/logs/history.json CHANGED
@@ -323,5 +323,18 @@
323
  "validation_loss": 0.2274281629114242,
324
  "validation_macro_f1": 0.9246719378407008,
325
  "validation_weighted_f1": 0.9255473513328624
 
 
 
 
 
 
 
 
 
 
 
 
 
326
  }
327
  ]
 
323
  "validation_loss": 0.2274281629114242,
324
  "validation_macro_f1": 0.9246719378407008,
325
  "validation_weighted_f1": 0.9255473513328624
326
+ },
327
+ {
328
+ "epoch": 25,
329
+ "epoch_seconds": 1160.5145826339722,
330
+ "global_step": 1274,
331
+ "gpu_peak_allocated_gib": 9.969139099121094,
332
+ "learning_rate": 0.0001,
333
+ "timestamp_utc": "2026-10-25T13:35:58.799441+00:00",
334
+ "train_loss": 0.4029264138669384,
335
+ "validation_accuracy": 0.927038626609442,
336
+ "validation_loss": 0.21226682162274293,
337
+ "validation_macro_f1": 0.9243063502544927,
338
+ "validation_weighted_f1": 0.9262645001176035
339
  }
340
  ]
runs/cars_issynth_20261025_seed0_support1/logs/metrics.csv CHANGED
@@ -24,3 +24,4 @@ epoch,global_step,train_loss,validation_loss,validation_accuracy,validation_macr
24
  22,1127,0.457498616588359,0.21050725348335259,0.927038626609442,0.9230498404121802,0.9252958100451424,0.0001,1155.00590467453,9.738087177276611,2026-10-25T12:38:19.995127+00:00
25
  23,1176,0.4228757975052815,0.2188762431843621,0.923973022685469,0.9199896424998306,0.9218467603007753,0.0001,1132.4107205867767,9.64440631866455,2026-10-25T12:57:21.399031+00:00
26
  24,1225,0.42196226703877354,0.2274281629114242,0.9264255058246474,0.9246719378407008,0.9255473513328624,0.0001,1137.8397467136383,9.553625583648682,2026-10-25T13:16:28.067599+00:00
 
 
24
  22,1127,0.457498616588359,0.21050725348335259,0.927038626609442,0.9230498404121802,0.9252958100451424,0.0001,1155.00590467453,9.738087177276611,2026-10-25T12:38:19.995127+00:00
25
  23,1176,0.4228757975052815,0.2188762431843621,0.923973022685469,0.9199896424998306,0.9218467603007753,0.0001,1132.4107205867767,9.64440631866455,2026-10-25T12:57:21.399031+00:00
26
  24,1225,0.42196226703877354,0.2274281629114242,0.9264255058246474,0.9246719378407008,0.9255473513328624,0.0001,1137.8397467136383,9.553625583648682,2026-10-25T13:16:28.067599+00:00
27
+ 25,1274,0.4029264138669384,0.21226682162274293,0.927038626609442,0.9243063502544927,0.9262645001176035,0.0001,1160.5145826339722,9.969139099121094,2026-10-25T13:35:58.799441+00:00
runs/cars_issynth_20261025_seed0_support1/logs/status.json CHANGED
@@ -1,20 +1,20 @@
1
  {
2
  "best_epoch": 7,
3
  "best_validation_accuracy": 0.9325567136725935,
4
- "current_epoch": 25,
5
- "elapsed_seconds": 28536.712498426437,
6
  "error": null,
7
- "estimated_remaining_seconds": 17122.02749905586,
8
- "global_step": 1225,
9
  "gpu_memory_allocated_gib": 4.065773963928223,
10
  "gpu_memory_reserved_gib": 14.94140625,
11
- "latest_checkpoint_timestamp": 1792934188.18809,
12
- "latest_hf_sync_timestamp": "2026-10-25T13:16:28.586594+00:00",
13
- "latest_train_loss": 0.42196226703877354,
14
- "latest_validation_accuracy": 0.9264255058246474,
15
- "latest_validation_loss": 0.2274281629114242,
16
  "run_id": "cars_issynth_20261025_seed0_support1",
17
  "status": "RUNNING",
18
  "total_epochs": 40,
19
- "updated_at_utc": "2026-10-25T13:16:28.587466+00:00"
20
  }
 
1
  {
2
  "best_epoch": 7,
3
  "best_validation_accuracy": 0.9325567136725935,
4
+ "current_epoch": 26,
5
+ "elapsed_seconds": 29697.22708106041,
6
  "error": null,
7
+ "estimated_remaining_seconds": 15990.814582109451,
8
+ "global_step": 1274,
9
  "gpu_memory_allocated_gib": 4.065773963928223,
10
  "gpu_memory_reserved_gib": 14.94140625,
11
+ "latest_checkpoint_timestamp": 1792935358.9162202,
12
+ "latest_hf_sync_timestamp": "2026-10-25T13:35:59.207871+00:00",
13
+ "latest_train_loss": 0.4029264138669384,
14
+ "latest_validation_accuracy": 0.927038626609442,
15
+ "latest_validation_loss": 0.21226682162274293,
16
  "run_id": "cars_issynth_20261025_seed0_support1",
17
  "status": "RUNNING",
18
  "total_epochs": 40,
19
+ "updated_at_utc": "2026-10-25T13:35:59.208650+00:00"
20
  }
runs/cars_issynth_20261025_seed0_support1/logs/supervisor.log CHANGED
@@ -5747,3 +5747,596 @@ I0000 00:00:1792915733.588368 2431 port.cc:153] oneDNN custom operations are
5747
 
5748
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB
5749
 
5750
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB
5751
 
5752
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5753
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
 
 
 
5754
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5755
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
 
 
 
 
 
5756
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5757
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5758
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
 
 
5759
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
 
 
 
5760
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5761
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
 
 
 
 
 
5762
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5763
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5764
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
 
 
5765
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
 
 
 
5766
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5767
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
 
 
 
 
 
5768
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5769
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5770
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
 
 
5771
  ...kpoints/current/latest.pt: 8%|▊ | 5.61MB / 67.6MB 
 
 
 
5772
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5773
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
 
 
5774
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5775
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5776
  ...nts/periodic/epoch_014.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5777
  ...kpoints/current/latest.pt: 39%|███▉ | 26.4MB / 67.6MB 
 
 
 
5778
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5779
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
 
 
5780
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5781
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5782
  ...nts/periodic/epoch_014.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5783
  ...kpoints/current/latest.pt: 57%|█████▋ | 38.8MB / 67.6MB 
 
 
 
5784
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5785
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5786
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5787
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5788
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5789
  ...kpoints/current/latest.pt: 57%|█████▋ | 38.8MB / 67.6MB 
 
 
 
5790
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5791
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5792
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5793
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5794
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5795
  ...kpoints/current/latest.pt: 64%|██████▍ | 43.3MB / 67.6MB 
 
 
 
5796
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5797
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5798
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5799
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5800
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5801
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5802
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5803
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5804
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5805
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5806
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5807
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5808
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5809
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5810
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5811
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5812
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5813
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5814
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5815
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5816
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5817
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5818
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5819
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5820
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5821
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5822
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5823
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5824
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5825
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5826
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5827
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5828
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5829
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5830
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5831
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
 
 
 
5832
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5833
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5834
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5835
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5836
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
5837
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5838
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5839
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5840
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5841
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5842
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5843
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5844
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5845
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5846
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5847
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5848
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
 
 
 
5849
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5850
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5851
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5852
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5853
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5854
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
 
 
 
5855
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
5856
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
5857
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
5858
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
5859
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
5860
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
 
 
 
 
 
 
 
 
 
 
 
5861
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB
 
5862
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB
 
5863
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB
 
5864
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB
 
5865
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB
 
5866
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB
 
5747
 
5748
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB
5749
 
5750
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB
5751
 
5752
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB
5753
+ [2026-10-25T13:16:32.202830+00:00] epoch=24 hf_snapshot_synced=true
5754
+ [2026-10-25T13:16:55.973329+00:00] step=1226 batch=1/49 loss=0.387827 real=0.050080 synth=1.738815 lr=0.0001 gpu_alloc_gib=4.085
5755
+ [2026-10-25T13:17:19.523866+00:00] step=1227 batch=2/49 loss=0.425157 real=0.061391 synth=1.880219 lr=0.0001 gpu_alloc_gib=4.085
5756
+ [2026-10-25T13:17:41.372117+00:00] step=1228 batch=3/49 loss=0.426706 real=0.053305 synth=1.920311 lr=0.0001 gpu_alloc_gib=4.085
5757
+ [2026-10-25T13:18:04.347806+00:00] step=1229 batch=4/49 loss=0.400076 real=0.073255 synth=1.707359 lr=0.0001 gpu_alloc_gib=4.085
5758
+ [2026-10-25T13:18:25.130531+00:00] step=1230 batch=5/49 loss=0.378534 real=0.061018 synth=1.648602 lr=0.0001 gpu_alloc_gib=4.085
5759
+ [2026-10-25T13:18:49.571856+00:00] step=1231 batch=6/49 loss=0.383653 real=0.057846 synth=1.686882 lr=0.0001 gpu_alloc_gib=4.085
5760
+ [2026-10-25T13:19:09.182774+00:00] step=1232 batch=7/49 loss=0.422298 real=0.064301 synth=1.854286 lr=0.0001 gpu_alloc_gib=4.085
5761
+ [2026-10-25T13:19:33.181061+00:00] step=1233 batch=8/49 loss=0.373771 real=0.083475 synth=1.534954 lr=0.0001 gpu_alloc_gib=4.085
5762
+ [2026-10-25T13:19:52.373836+00:00] step=1234 batch=9/49 loss=0.403522 real=0.066181 synth=1.752888 lr=0.0001 gpu_alloc_gib=4.085
5763
+ [2026-10-25T13:20:13.142265+00:00] step=1235 batch=10/49 loss=0.418994 real=0.073065 synth=1.802711 lr=0.0001 gpu_alloc_gib=4.085
5764
+ [2026-10-25T13:20:36.065442+00:00] step=1236 batch=11/49 loss=0.387056 real=0.077922 synth=1.623592 lr=0.0001 gpu_alloc_gib=4.085
5765
+ [2026-10-25T13:20:56.126303+00:00] step=1237 batch=12/49 loss=0.400108 real=0.069202 synth=1.723730 lr=0.0001 gpu_alloc_gib=4.085
5766
+ [2026-10-25T13:21:16.583735+00:00] step=1238 batch=13/49 loss=0.436043 real=0.067789 synth=1.909058 lr=0.0001 gpu_alloc_gib=4.085
5767
+ [2026-10-25T13:21:37.467764+00:00] step=1239 batch=14/49 loss=0.456755 real=0.062835 synth=2.032438 lr=0.0001 gpu_alloc_gib=4.085
5768
+ [2026-10-25T13:21:55.396759+00:00] step=1240 batch=15/49 loss=0.368669 real=0.061006 synth=1.599323 lr=0.0001 gpu_alloc_gib=4.085
5769
+ [2026-10-25T13:22:17.602090+00:00] step=1241 batch=16/49 loss=0.407259 real=0.063757 synth=1.781265 lr=0.0001 gpu_alloc_gib=4.085
5770
+ [2026-10-25T13:22:39.269606+00:00] step=1242 batch=17/49 loss=0.301915 real=0.050290 synth=1.308414 lr=0.0001 gpu_alloc_gib=4.085
5771
+ [2026-10-25T13:22:59.569492+00:00] step=1243 batch=18/49 loss=0.427101 real=0.057976 synth=1.903599 lr=0.0001 gpu_alloc_gib=4.085
5772
+ [2026-10-25T13:23:18.995774+00:00] step=1244 batch=19/49 loss=0.436132 real=0.054398 synth=1.963066 lr=0.0001 gpu_alloc_gib=4.085
5773
+ [2026-10-25T13:23:46.813223+00:00] step=1245 batch=20/49 loss=0.467940 real=0.059570 synth=2.101418 lr=0.0001 gpu_alloc_gib=4.085
5774
+ [2026-10-25T13:24:09.000036+00:00] step=1246 batch=21/49 loss=0.386182 real=0.088748 synth=1.575920 lr=0.0001 gpu_alloc_gib=4.085
5775
+ [2026-10-25T13:24:31.374995+00:00] step=1247 batch=22/49 loss=0.458675 real=0.067333 synth=2.024040 lr=0.0001 gpu_alloc_gib=4.085
5776
+ [2026-10-25T13:24:53.163661+00:00] step=1248 batch=23/49 loss=0.332007 real=0.050366 synth=1.458570 lr=0.0001 gpu_alloc_gib=4.085
5777
+ [2026-10-25T13:25:14.996185+00:00] step=1249 batch=24/49 loss=0.328479 real=0.057549 synth=1.412197 lr=0.0001 gpu_alloc_gib=4.085
5778
+ [2026-10-25T13:25:37.969225+00:00] step=1250 batch=25/49 loss=0.400305 real=0.060694 synth=1.758751 lr=0.0001 gpu_alloc_gib=4.085
5779
+ [2026-10-25T13:25:57.721159+00:00] step=1251 batch=26/49 loss=0.361899 real=0.058741 synth=1.574530 lr=0.0001 gpu_alloc_gib=4.085
5780
+ [2026-10-25T13:26:20.264010+00:00] step=1252 batch=27/49 loss=0.376392 real=0.062050 synth=1.633762 lr=0.0001 gpu_alloc_gib=4.085
5781
+ [2026-10-25T13:26:40.094676+00:00] step=1253 batch=28/49 loss=0.404741 real=0.048239 synth=1.830751 lr=0.0001 gpu_alloc_gib=4.085
5782
+ [2026-10-25T13:27:02.378043+00:00] step=1254 batch=29/49 loss=0.405361 real=0.053226 synth=1.813902 lr=0.0001 gpu_alloc_gib=4.085
5783
+ [2026-10-25T13:27:23.560977+00:00] step=1255 batch=30/49 loss=0.427720 real=0.078431 synth=1.824875 lr=0.0001 gpu_alloc_gib=4.085
5784
+ [2026-10-25T13:27:47.435020+00:00] step=1256 batch=31/49 loss=0.432324 real=0.075218 synth=1.860746 lr=0.0001 gpu_alloc_gib=4.085
5785
+ [2026-10-25T13:28:12.976325+00:00] step=1257 batch=32/49 loss=0.344009 real=0.058304 synth=1.486832 lr=0.0001 gpu_alloc_gib=4.085
5786
+ [2026-10-25T13:28:34.117949+00:00] step=1258 batch=33/49 loss=0.400919 real=0.059367 synth=1.767128 lr=0.0001 gpu_alloc_gib=4.085
5787
+ [2026-10-25T13:28:51.286680+00:00] step=1259 batch=34/49 loss=0.415829 real=0.055393 synth=1.857574 lr=0.0001 gpu_alloc_gib=4.085
5788
+ [2026-10-25T13:29:11.232333+00:00] step=1260 batch=35/49 loss=0.443843 real=0.084138 synth=1.882662 lr=0.0001 gpu_alloc_gib=4.085
5789
+ [2026-10-25T13:29:31.612503+00:00] step=1261 batch=36/49 loss=0.442854 real=0.057862 synth=1.982819 lr=0.0001 gpu_alloc_gib=4.085
5790
+ [2026-10-25T13:29:51.653033+00:00] step=1262 batch=37/49 loss=0.438326 real=0.087075 synth=1.843330 lr=0.0001 gpu_alloc_gib=4.085
5791
+ [2026-10-25T13:30:14.923546+00:00] step=1263 batch=38/49 loss=0.431177 real=0.067043 synth=1.887712 lr=0.0001 gpu_alloc_gib=4.085
5792
+ [2026-10-25T13:30:39.027680+00:00] step=1264 batch=39/49 loss=0.413049 real=0.050589 synth=1.862890 lr=0.0001 gpu_alloc_gib=4.085
5793
+ [2026-10-25T13:31:01.746042+00:00] step=1265 batch=40/49 loss=0.421322 real=0.056847 synth=1.879223 lr=0.0001 gpu_alloc_gib=4.085
5794
+ [2026-10-25T13:31:22.458052+00:00] step=1266 batch=41/49 loss=0.393114 real=0.078235 synth=1.652634 lr=0.0001 gpu_alloc_gib=4.085
5795
+ [2026-10-25T13:31:43.606107+00:00] step=1267 batch=42/49 loss=0.421828 real=0.055216 synth=1.888275 lr=0.0001 gpu_alloc_gib=4.085
5796
+ [2026-10-25T13:32:03.878642+00:00] step=1268 batch=43/49 loss=0.357323 real=0.052651 synth=1.576012 lr=0.0001 gpu_alloc_gib=4.085
5797
+ [2026-10-25T13:32:26.466815+00:00] step=1269 batch=44/49 loss=0.420117 real=0.047214 synth=1.911732 lr=0.0001 gpu_alloc_gib=4.085
5798
+ [2026-10-25T13:32:45.117525+00:00] step=1270 batch=45/49 loss=0.475344 real=0.062787 synth=2.125572 lr=0.0001 gpu_alloc_gib=4.085
5799
+ [2026-10-25T13:33:04.491361+00:00] step=1271 batch=46/49 loss=0.394748 real=0.050778 synth=1.770626 lr=0.0001 gpu_alloc_gib=4.085
5800
+ [2026-10-25T13:33:21.956448+00:00] step=1272 batch=47/49 loss=0.352423 real=0.056441 synth=1.536350 lr=0.0001 gpu_alloc_gib=4.085
5801
+ [2026-10-25T13:33:42.118352+00:00] step=1273 batch=48/49 loss=0.337394 real=0.048782 synth=1.491844 lr=0.0001 gpu_alloc_gib=4.085
5802
+ [2026-10-25T13:34:00.679112+00:00] step=1274 batch=49/49 loss=0.416173 real=0.055770 synth=1.857784 lr=0.0001 gpu_alloc_gib=4.085
5803
+ [2026-10-25T13:35:59.207970+00:00] epoch=25 val_accuracy=0.927039 val_macro_f1=0.924306 best=0.932557 sync=pending
5804
+
5805
+
5806
+
5807
+
5808
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
5809
+
5810
+
5811
+
5812
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5813
+
5814
+
5815
+
5816
+
5817
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
5818
+
5819
+
5820
+
5821
+
5822
+
5823
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5824
+
5825
+
5826
+
5827
+
5828
+
5829
+
5830
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5831
+
5832
+
5833
+
5834
+
5835
+
5836
+
5837
+
5838
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
5839
+
5840
+
5841
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
5842
+
5843
+
5844
+
5845
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5846
+
5847
+
5848
+
5849
+
5850
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
5851
+
5852
+
5853
+
5854
+
5855
+
5856
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5857
+
5858
+
5859
+
5860
+
5861
+
5862
+
5863
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5864
+
5865
+
5866
+
5867
+
5868
+
5869
+
5870
+
5871
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
5872
+
5873
+
5874
  ...kpoints/current/latest.pt: 0%| | 60.7kB / 67.6MB 
5875
+
5876
+
5877
+
5878
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5879
+
5880
+
5881
+
5882
+
5883
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.4MB / 67.6MB 
5884
+
5885
+
5886
+
5887
+
5888
+
5889
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5890
+
5891
+
5892
+
5893
+
5894
+
5895
+
5896
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5897
+
5898
+
5899
+
5900
+
5901
+
5902
+
5903
+
5904
  ...nts/periodic/epoch_014.pt: 99%|█████████▉| 67.2MB / 67.6MB 
5905
+
5906
+
5907
  ...kpoints/current/latest.pt: 8%|▊ | 5.61MB / 67.6MB 
5908
+
5909
+
5910
+
5911
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5912
+
5913
+
5914
+
5915
+
5916
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB 
5917
+
5918
+
5919
+
5920
+
5921
+
5922
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5923
+
5924
+
5925
+
5926
+
5927
+
5928
+
5929
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5930
+
5931
+
5932
+
5933
+
5934
+
5935
+
5936
+
5937
  ...nts/periodic/epoch_014.pt: 100%|█████████▉| 67.6MB / 67.6MB 
5938
+
5939
+
5940
+
5941
  ...kpoints/current/latest.pt: 39%|███▉ | 26.4MB / 67.6MB 
5942
+
5943
+
5944
+
5945
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5946
+
5947
+
5948
+
5949
+
5950
  ...nts/periodic/epoch_009.pt: 100%|█████████▉| 67.6MB / 67.6MB 
5951
+
5952
+
5953
+
5954
+
5955
+
5956
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5957
+
5958
+
5959
+
5960
+
5961
+
5962
+
5963
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5964
+
5965
+
5966
+
5967
+
5968
+
5969
+
5970
+
5971
  ...nts/periodic/epoch_014.pt: 100%|█████████▉| 67.6MB / 67.6MB 
5972
+
5973
+
5974
+
5975
  ...kpoints/current/latest.pt: 57%|█████▋ | 38.8MB / 67.6MB 
5976
+
5977
+
5978
+
5979
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
5980
+
5981
+
5982
+
5983
+
5984
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
5985
+
5986
+
5987
+
5988
+
5989
+
5990
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
5991
+
5992
+
5993
+
5994
+
5995
+
5996
+
5997
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
5998
+
5999
+
6000
+
6001
+
6002
+
6003
+
6004
+
6005
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6006
+
6007
+
6008
+
6009
  ...kpoints/current/latest.pt: 57%|█████▋ | 38.8MB / 67.6MB 
6010
+
6011
+
6012
+
6013
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6014
+
6015
+
6016
+
6017
+
6018
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6019
+
6020
+
6021
+
6022
+
6023
+
6024
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6025
+
6026
+
6027
+
6028
+
6029
+
6030
+
6031
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6032
+
6033
+
6034
+
6035
+
6036
+
6037
+
6038
+
6039
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6040
+
6041
+
6042
  ...kpoints/current/latest.pt: 64%|██████▍ | 43.3MB / 67.6MB 
6043
+
6044
+
6045
+
6046
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6047
+
6048
+
6049
+
6050
+
6051
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6052
+
6053
+
6054
+
6055
+
6056
+
6057
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6058
+
6059
+
6060
+
6061
+
6062
+
6063
+
6064
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6065
+
6066
+
6067
+
6068
+
6069
+
6070
+
6071
+
6072
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6073
+
6074
+
6075
+
6076
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6077
+
6078
+
6079
+
6080
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6081
+
6082
+
6083
+
6084
+
6085
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6086
+
6087
+
6088
+
6089
+
6090
+
6091
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6092
+
6093
+
6094
+
6095
+
6096
+
6097
+
6098
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6099
+
6100
+
6101
+
6102
+
6103
+
6104
+
6105
+
6106
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6107
+
6108
+
6109
+
6110
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6111
+
6112
+
6113
+
6114
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6115
+
6116
+
6117
+
6118
+
6119
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6120
+
6121
+
6122
+
6123
+
6124
+
6125
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6126
+
6127
+
6128
+
6129
+
6130
+
6131
+
6132
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6133
+
6134
+
6135
+
6136
+
6137
+
6138
+
6139
+
6140
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6141
+
6142
+
6143
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6144
+
6145
+
6146
+
6147
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6148
+
6149
+
6150
+
6151
+
6152
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6153
+
6154
+
6155
+
6156
+
6157
+
6158
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6159
+
6160
+
6161
+
6162
+
6163
+
6164
+
6165
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6166
+
6167
+
6168
+
6169
+
6170
+
6171
+
6172
+
6173
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6174
+
6175
+
6176
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6177
+
6178
+
6179
+
6180
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6181
+
6182
+
6183
+
6184
+
6185
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6186
+
6187
+
6188
+
6189
+
6190
+
6191
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6192
+
6193
+
6194
+
6195
+
6196
+
6197
+
6198
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6199
+
6200
+
6201
+
6202
+
6203
+
6204
+
6205
+
6206
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6207
+
6208
+
6209
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6210
+
6211
+
6212
+
6213
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6214
+
6215
+
6216
+
6217
+
6218
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6219
+
6220
+
6221
+
6222
+
6223
+
6224
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6225
+
6226
+
6227
+
6228
+
6229
+
6230
+
6231
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6232
+
6233
+
6234
+
6235
+
6236
+
6237
+
6238
+
6239
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6240
+
6241
+
6242
  ...kpoints/current/latest.pt: 100%|█████████▉| 67.6MB / 67.6MB 
6243
+
6244
+
6245
+
6246
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6247
+
6248
+
6249
+
6250
+
6251
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6252
+
6253
+
6254
+
6255
+
6256
+
6257
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6258
+
6259
+
6260
+
6261
+
6262
+
6263
+
6264
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6265
+
6266
+
6267
+
6268
+
6269
+
6270
+
6271
+
6272
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6273
+
6274
+
6275
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
6276
+
6277
+
6278
+
6279
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6280
+
6281
+
6282
+
6283
+
6284
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6285
+
6286
+
6287
+
6288
+
6289
+
6290
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6291
+
6292
+
6293
+
6294
+
6295
+
6296
+
6297
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6298
+
6299
+
6300
+
6301
+
6302
+
6303
+
6304
+
6305
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6306
+
6307
+
6308
+
6309
+
6310
+
6311
+
6312
+
6313
+
6314
+
6315
+
6316
+
6317
+
6318
+
6319
+
6320
+
6321
+
6322
+
6323
+
6324
+
6325
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
6326
+
6327
+
6328
+
6329
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6330
+
6331
+
6332
+
6333
+
6334
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6335
+
6336
+
6337
+
6338
+
6339
+
6340
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6341
+
6342
+
6343
+
6344
+
6345
+
6346
+
6347
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6348
+
6349
+
6350
+
6351
+
6352
+
6353
+
6354
+
6355
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6356
+
6357
+
6358
+
6359
+
6360
+
6361
+
6362
+
6363
+
6364
+
6365
+
6366
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
6367
+
6368
+
6369
+
6370
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6371
+
6372
+
6373
+
6374
+
6375
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6376
+
6377
+
6378
+
6379
+
6380
+
6381
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6382
+
6383
+
6384
+
6385
+
6386
+
6387
+
6388
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6389
+
6390
+
6391
+
6392
+
6393
+
6394
+
6395
+
6396
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6397
+
6398
+
6399
+
6400
+
6401
+
6402
+
6403
+
6404
+
6405
+
6406
+
6407
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB 
6408
+
6409
+
6410
+
6411
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB 
6412
+
6413
+
6414
+
6415
+
6416
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB 
6417
+
6418
+
6419
+
6420
+
6421
+
6422
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB 
6423
+
6424
+
6425
+
6426
+
6427
+
6428
+
6429
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB 
6430
+
6431
+
6432
+
6433
+
6434
+
6435
+
6436
+
6437
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB 
6438
+
6439
+
6440
+
6441
+
6442
+
6443
+
6444
+
6445
+
6446
+
6447
+
6448
+
6449
  ...kpoints/current/latest.pt: 100%|██████████| 67.6MB / 67.6MB
6450
+
6451
  ...nts/periodic/epoch_019.pt: 100%|██████████| 67.6MB / 67.6MB
6452
+
6453
  ...nts/periodic/epoch_009.pt: 100%|██████████| 67.6MB / 67.6MB
6454
+
6455
  ...nts/periodic/epoch_024.pt: 100%|██████████| 67.6MB / 67.6MB
6456
+
6457
  .../checkpoints/best/best.pt: 100%|██████████| 67.6MB / 67.6MB
6458
+
6459
  ...nts/periodic/epoch_014.pt: 100%|██████████| 67.6MB / 67.6MB
runs/cars_issynth_20261025_seed0_support1/logs/train.log CHANGED
@@ -1274,3 +1274,54 @@
1274
  [2026-10-25T13:14:09.931111+00:00] step=1224 batch=48/49 loss=0.479095 real=0.084579 synth=2.057159 lr=0.0001 gpu_alloc_gib=4.085
1275
  [2026-10-25T13:14:30.795062+00:00] step=1225 batch=49/49 loss=0.487568 real=0.074060 synth=2.141596 lr=0.0001 gpu_alloc_gib=4.085
1276
  [2026-10-25T13:16:28.586704+00:00] epoch=24 val_accuracy=0.926426 val_macro_f1=0.924672 best=0.932557 sync=pending
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1274
  [2026-10-25T13:14:09.931111+00:00] step=1224 batch=48/49 loss=0.479095 real=0.084579 synth=2.057159 lr=0.0001 gpu_alloc_gib=4.085
1275
  [2026-10-25T13:14:30.795062+00:00] step=1225 batch=49/49 loss=0.487568 real=0.074060 synth=2.141596 lr=0.0001 gpu_alloc_gib=4.085
1276
  [2026-10-25T13:16:28.586704+00:00] epoch=24 val_accuracy=0.926426 val_macro_f1=0.924672 best=0.932557 sync=pending
1277
+ [2026-10-25T13:16:32.202830+00:00] epoch=24 hf_snapshot_synced=true
1278
+ [2026-10-25T13:16:55.973329+00:00] step=1226 batch=1/49 loss=0.387827 real=0.050080 synth=1.738815 lr=0.0001 gpu_alloc_gib=4.085
1279
+ [2026-10-25T13:17:19.523866+00:00] step=1227 batch=2/49 loss=0.425157 real=0.061391 synth=1.880219 lr=0.0001 gpu_alloc_gib=4.085
1280
+ [2026-10-25T13:17:41.372117+00:00] step=1228 batch=3/49 loss=0.426706 real=0.053305 synth=1.920311 lr=0.0001 gpu_alloc_gib=4.085
1281
+ [2026-10-25T13:18:04.347806+00:00] step=1229 batch=4/49 loss=0.400076 real=0.073255 synth=1.707359 lr=0.0001 gpu_alloc_gib=4.085
1282
+ [2026-10-25T13:18:25.130531+00:00] step=1230 batch=5/49 loss=0.378534 real=0.061018 synth=1.648602 lr=0.0001 gpu_alloc_gib=4.085
1283
+ [2026-10-25T13:18:49.571856+00:00] step=1231 batch=6/49 loss=0.383653 real=0.057846 synth=1.686882 lr=0.0001 gpu_alloc_gib=4.085
1284
+ [2026-10-25T13:19:09.182774+00:00] step=1232 batch=7/49 loss=0.422298 real=0.064301 synth=1.854286 lr=0.0001 gpu_alloc_gib=4.085
1285
+ [2026-10-25T13:19:33.181061+00:00] step=1233 batch=8/49 loss=0.373771 real=0.083475 synth=1.534954 lr=0.0001 gpu_alloc_gib=4.085
1286
+ [2026-10-25T13:19:52.373836+00:00] step=1234 batch=9/49 loss=0.403522 real=0.066181 synth=1.752888 lr=0.0001 gpu_alloc_gib=4.085
1287
+ [2026-10-25T13:20:13.142265+00:00] step=1235 batch=10/49 loss=0.418994 real=0.073065 synth=1.802711 lr=0.0001 gpu_alloc_gib=4.085
1288
+ [2026-10-25T13:20:36.065442+00:00] step=1236 batch=11/49 loss=0.387056 real=0.077922 synth=1.623592 lr=0.0001 gpu_alloc_gib=4.085
1289
+ [2026-10-25T13:20:56.126303+00:00] step=1237 batch=12/49 loss=0.400108 real=0.069202 synth=1.723730 lr=0.0001 gpu_alloc_gib=4.085
1290
+ [2026-10-25T13:21:16.583735+00:00] step=1238 batch=13/49 loss=0.436043 real=0.067789 synth=1.909058 lr=0.0001 gpu_alloc_gib=4.085
1291
+ [2026-10-25T13:21:37.467764+00:00] step=1239 batch=14/49 loss=0.456755 real=0.062835 synth=2.032438 lr=0.0001 gpu_alloc_gib=4.085
1292
+ [2026-10-25T13:21:55.396759+00:00] step=1240 batch=15/49 loss=0.368669 real=0.061006 synth=1.599323 lr=0.0001 gpu_alloc_gib=4.085
1293
+ [2026-10-25T13:22:17.602090+00:00] step=1241 batch=16/49 loss=0.407259 real=0.063757 synth=1.781265 lr=0.0001 gpu_alloc_gib=4.085
1294
+ [2026-10-25T13:22:39.269606+00:00] step=1242 batch=17/49 loss=0.301915 real=0.050290 synth=1.308414 lr=0.0001 gpu_alloc_gib=4.085
1295
+ [2026-10-25T13:22:59.569492+00:00] step=1243 batch=18/49 loss=0.427101 real=0.057976 synth=1.903599 lr=0.0001 gpu_alloc_gib=4.085
1296
+ [2026-10-25T13:23:18.995774+00:00] step=1244 batch=19/49 loss=0.436132 real=0.054398 synth=1.963066 lr=0.0001 gpu_alloc_gib=4.085
1297
+ [2026-10-25T13:23:46.813223+00:00] step=1245 batch=20/49 loss=0.467940 real=0.059570 synth=2.101418 lr=0.0001 gpu_alloc_gib=4.085
1298
+ [2026-10-25T13:24:09.000036+00:00] step=1246 batch=21/49 loss=0.386182 real=0.088748 synth=1.575920 lr=0.0001 gpu_alloc_gib=4.085
1299
+ [2026-10-25T13:24:31.374995+00:00] step=1247 batch=22/49 loss=0.458675 real=0.067333 synth=2.024040 lr=0.0001 gpu_alloc_gib=4.085
1300
+ [2026-10-25T13:24:53.163661+00:00] step=1248 batch=23/49 loss=0.332007 real=0.050366 synth=1.458570 lr=0.0001 gpu_alloc_gib=4.085
1301
+ [2026-10-25T13:25:14.996185+00:00] step=1249 batch=24/49 loss=0.328479 real=0.057549 synth=1.412197 lr=0.0001 gpu_alloc_gib=4.085
1302
+ [2026-10-25T13:25:37.969225+00:00] step=1250 batch=25/49 loss=0.400305 real=0.060694 synth=1.758751 lr=0.0001 gpu_alloc_gib=4.085
1303
+ [2026-10-25T13:25:57.721159+00:00] step=1251 batch=26/49 loss=0.361899 real=0.058741 synth=1.574530 lr=0.0001 gpu_alloc_gib=4.085
1304
+ [2026-10-25T13:26:20.264010+00:00] step=1252 batch=27/49 loss=0.376392 real=0.062050 synth=1.633762 lr=0.0001 gpu_alloc_gib=4.085
1305
+ [2026-10-25T13:26:40.094676+00:00] step=1253 batch=28/49 loss=0.404741 real=0.048239 synth=1.830751 lr=0.0001 gpu_alloc_gib=4.085
1306
+ [2026-10-25T13:27:02.378043+00:00] step=1254 batch=29/49 loss=0.405361 real=0.053226 synth=1.813902 lr=0.0001 gpu_alloc_gib=4.085
1307
+ [2026-10-25T13:27:23.560977+00:00] step=1255 batch=30/49 loss=0.427720 real=0.078431 synth=1.824875 lr=0.0001 gpu_alloc_gib=4.085
1308
+ [2026-10-25T13:27:47.435020+00:00] step=1256 batch=31/49 loss=0.432324 real=0.075218 synth=1.860746 lr=0.0001 gpu_alloc_gib=4.085
1309
+ [2026-10-25T13:28:12.976325+00:00] step=1257 batch=32/49 loss=0.344009 real=0.058304 synth=1.486832 lr=0.0001 gpu_alloc_gib=4.085
1310
+ [2026-10-25T13:28:34.117949+00:00] step=1258 batch=33/49 loss=0.400919 real=0.059367 synth=1.767128 lr=0.0001 gpu_alloc_gib=4.085
1311
+ [2026-10-25T13:28:51.286680+00:00] step=1259 batch=34/49 loss=0.415829 real=0.055393 synth=1.857574 lr=0.0001 gpu_alloc_gib=4.085
1312
+ [2026-10-25T13:29:11.232333+00:00] step=1260 batch=35/49 loss=0.443843 real=0.084138 synth=1.882662 lr=0.0001 gpu_alloc_gib=4.085
1313
+ [2026-10-25T13:29:31.612503+00:00] step=1261 batch=36/49 loss=0.442854 real=0.057862 synth=1.982819 lr=0.0001 gpu_alloc_gib=4.085
1314
+ [2026-10-25T13:29:51.653033+00:00] step=1262 batch=37/49 loss=0.438326 real=0.087075 synth=1.843330 lr=0.0001 gpu_alloc_gib=4.085
1315
+ [2026-10-25T13:30:14.923546+00:00] step=1263 batch=38/49 loss=0.431177 real=0.067043 synth=1.887712 lr=0.0001 gpu_alloc_gib=4.085
1316
+ [2026-10-25T13:30:39.027680+00:00] step=1264 batch=39/49 loss=0.413049 real=0.050589 synth=1.862890 lr=0.0001 gpu_alloc_gib=4.085
1317
+ [2026-10-25T13:31:01.746042+00:00] step=1265 batch=40/49 loss=0.421322 real=0.056847 synth=1.879223 lr=0.0001 gpu_alloc_gib=4.085
1318
+ [2026-10-25T13:31:22.458052+00:00] step=1266 batch=41/49 loss=0.393114 real=0.078235 synth=1.652634 lr=0.0001 gpu_alloc_gib=4.085
1319
+ [2026-10-25T13:31:43.606107+00:00] step=1267 batch=42/49 loss=0.421828 real=0.055216 synth=1.888275 lr=0.0001 gpu_alloc_gib=4.085
1320
+ [2026-10-25T13:32:03.878642+00:00] step=1268 batch=43/49 loss=0.357323 real=0.052651 synth=1.576012 lr=0.0001 gpu_alloc_gib=4.085
1321
+ [2026-10-25T13:32:26.466815+00:00] step=1269 batch=44/49 loss=0.420117 real=0.047214 synth=1.911732 lr=0.0001 gpu_alloc_gib=4.085
1322
+ [2026-10-25T13:32:45.117525+00:00] step=1270 batch=45/49 loss=0.475344 real=0.062787 synth=2.125572 lr=0.0001 gpu_alloc_gib=4.085
1323
+ [2026-10-25T13:33:04.491361+00:00] step=1271 batch=46/49 loss=0.394748 real=0.050778 synth=1.770626 lr=0.0001 gpu_alloc_gib=4.085
1324
+ [2026-10-25T13:33:21.956448+00:00] step=1272 batch=47/49 loss=0.352423 real=0.056441 synth=1.536350 lr=0.0001 gpu_alloc_gib=4.085
1325
+ [2026-10-25T13:33:42.118352+00:00] step=1273 batch=48/49 loss=0.337394 real=0.048782 synth=1.491844 lr=0.0001 gpu_alloc_gib=4.085
1326
+ [2026-10-25T13:34:00.679112+00:00] step=1274 batch=49/49 loss=0.416173 real=0.055770 synth=1.857784 lr=0.0001 gpu_alloc_gib=4.085
1327
+ [2026-10-25T13:35:59.207970+00:00] epoch=25 val_accuracy=0.927039 val_macro_f1=0.924306 best=0.932557 sync=pending
runs/cars_issynth_20261025_seed0_support1/metrics/gpu.csv CHANGED
@@ -614,3 +614,42 @@ timestamp_utc,gpu_util_percent,memory_used_mib,memory_free_mib,temperature_c,pow
614
  2026-10-25T13:15:11.585079+00:00,93,16023,81230,47,282.94
615
  2026-10-25T13:15:41.631581+00:00,100,16023,81230,50,259.04
616
  2026-10-25T13:16:11.673397+00:00,94,16023,81230,48,296.77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
614
  2026-10-25T13:15:11.585079+00:00,93,16023,81230,47,282.94
615
  2026-10-25T13:15:41.631581+00:00,100,16023,81230,50,259.04
616
  2026-10-25T13:16:11.673397+00:00,94,16023,81230,48,296.77
617
+ 2026-10-25T13:16:41.719601+00:00,61,16023,81230,45,285.49
618
+ 2026-10-25T13:17:11.763884+00:00,100,16023,81230,48,309.52
619
+ 2026-10-25T13:17:41.804167+00:00,4,16023,81230,41,275.76
620
+ 2026-10-25T13:18:11.851903+00:00,58,16023,81230,45,239.87
621
+ 2026-10-25T13:18:41.893565+00:00,49,16023,81230,47,338.36
622
+ 2026-10-25T13:19:11.935532+00:00,66,16023,81230,44,242.09
623
+ 2026-10-25T13:19:41.982880+00:00,100,16023,81230,49,280.55
624
+ 2026-10-25T13:20:12.032177+00:00,64,16023,81230,48,298.89
625
+ 2026-10-25T13:20:42.070214+00:00,60,16023,81230,46,248.45
626
+ 2026-10-25T13:21:12.114783+00:00,64,16023,81230,47,349.82
627
+ 2026-10-25T13:21:42.153084+00:00,64,16023,81230,44,259.16
628
+ 2026-10-25T13:22:12.199402+00:00,100,16023,81230,48,341.79
629
+ 2026-10-25T13:22:42.239274+00:00,60,16023,81230,44,244.10
630
+ 2026-10-25T13:23:12.285332+00:00,100,16023,81230,48,346.68
631
+ 2026-10-25T13:23:42.336737+00:00,100,16023,81230,47,352.62
632
+ 2026-10-25T13:24:12.378548+00:00,61,16023,81230,45,261.46
633
+ 2026-10-25T13:24:42.424682+00:00,100,16023,81230,46,385.95
634
+ 2026-10-25T13:25:12.470193+00:00,2,16023,81230,48,323.78
635
+ 2026-10-25T13:25:42.519864+00:00,60,16023,81230,46,262.82
636
+ 2026-10-25T13:26:12.561658+00:00,100,16023,81230,48,362.87
637
+ 2026-10-25T13:26:42.600417+00:00,61,16023,81230,46,266.37
638
+ 2026-10-25T13:27:12.640063+00:00,75,16023,81230,49,345.28
639
+ 2026-10-25T13:27:42.684054+00:00,0,16023,81230,41,300.00
640
+ 2026-10-25T13:28:12.726156+00:00,95,16023,81230,49,285.78
641
+ 2026-10-25T13:28:42.775074+00:00,36,16023,81230,48,258.46
642
+ 2026-10-25T13:29:12.823215+00:00,67,16023,81230,45,228.74
643
+ 2026-10-25T13:29:42.867546+00:00,100,16023,81230,46,345.03
644
+ 2026-10-25T13:30:12.910799+00:00,96,16023,81230,48,297.93
645
+ 2026-10-25T13:30:42.951000+00:00,70,16023,81230,46,267.11
646
+ 2026-10-25T13:31:12.995727+00:00,100,16023,81230,47,309.75
647
+ 2026-10-25T13:31:43.037344+00:00,49,16023,81230,46,274.67
648
+ 2026-10-25T13:32:13.083411+00:00,100,16023,81230,46,356.60
649
+ 2026-10-25T13:32:43.127580+00:00,88,16023,81230,48,278.01
650
+ 2026-10-25T13:33:13.167577+00:00,3,16023,81230,41,285.86
651
+ 2026-10-25T13:33:43.203259+00:00,59,16023,81230,46,220.67
652
+ 2026-10-25T13:34:13.245168+00:00,3,16023,81230,38,187.07
653
+ 2026-10-25T13:34:43.293428+00:00,0,16023,81230,40,249.73
654
+ 2026-10-25T13:35:13.343916+00:00,100,16023,81230,49,320.51
655
+ 2026-10-25T13:35:43.394297+00:00,97,16023,81230,48,291.66
runs/cars_issynth_20261025_seed0_support1/plots/accuracy_curve.png CHANGED
runs/cars_issynth_20261025_seed0_support1/plots/f1_curve.png CHANGED
runs/cars_issynth_20261025_seed0_support1/plots/loss_curve.png CHANGED