Final: identity pair + 3 transformer seeds close all deferred significance pairs
Browse files- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/logs/metrics.json +211 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/logs/transformer_asa_3d.log +18 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/logs/metrics.json +477 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/logs/transformer_asa_4d.log +64 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/logs/metrics.json +477 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/logs/transformer_fa_3d.log +64 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/logs/metrics.json +344 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/logs/transformer_fa_4d.log +50 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s619.log +60 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s732.log +64 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s947.log +60 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/manifest.json +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/logs/metrics.json +211 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/logs/transformer_asa_2d.log +18 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/logs/metrics.json +249 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/logs/transformer_asa_2d.log +23 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/run_complete.json +5 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/best.pth +3 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/cost.json +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/logs/metrics.json +211 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/logs/transformer_asa_2d.log +21 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/plots/loss_curve.png +0 -0
- exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/run_complete.json +5 -0
- exp7_exp8_operator_study/final/results/errdump_FINAL.npz +3 -0
- exp7_exp8_operator_study/final/results/significance_FINAL_v3.txt +63 -0
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:de567d8449e77aec06c47fc7b095c968148ae43bd6edf94c98d8b0759796e7c3
|
| 3 |
+
size 4717723
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 9073566252288,
|
| 7 |
+
"flops_per_observation": 301577633.273108,
|
| 8 |
+
"epochs_run": 11,
|
| 9 |
+
"epochs_to_best": 1,
|
| 10 |
+
"optimizer_steps_total": 1056,
|
| 11 |
+
"optimizer_steps_to_best": 96,
|
| 12 |
+
"observations_total": 31771872,
|
| 13 |
+
"observations_to_best": 2888352,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 9581685962416128.0,
|
| 16 |
+
"train_flops_to_best": 871062360219648.0,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "asa_3d",
|
| 20 |
+
"seed": 619
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/logs/metrics.json
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.0780429457469533,
|
| 6 |
+
"val_loss": 0.8062640825907389,
|
| 7 |
+
"lr": 0.001,
|
| 8 |
+
"epoch_seconds": 97.5204930305481,
|
| 9 |
+
"train_seconds": 90.22,
|
| 10 |
+
"val_seconds": 7.3,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.8062640825907389,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 1.7155895233154297,
|
| 17 |
+
"grad_norm_max": 23.972929000854492,
|
| 18 |
+
"clip_fraction": 0.5416666865348816,
|
| 19 |
+
"gpu_peak_gb": 59.729
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.0363939053301389,
|
| 25 |
+
"val_loss": 0.8147425130009651,
|
| 26 |
+
"lr": 0.001,
|
| 27 |
+
"epoch_seconds": 95.36435794830322,
|
| 28 |
+
"train_seconds": 88.243,
|
| 29 |
+
"val_seconds": 7.121,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.8062640825907389,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.5741003751754761,
|
| 36 |
+
"grad_norm_max": 2.131016969680786,
|
| 37 |
+
"clip_fraction": 0.1041666716337204,
|
| 38 |
+
"gpu_peak_gb": 59.729
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.0333457585851041,
|
| 44 |
+
"val_loss": 0.8089850371082624,
|
| 45 |
+
"lr": 0.001,
|
| 46 |
+
"epoch_seconds": 95.4329195022583,
|
| 47 |
+
"train_seconds": 88.315,
|
| 48 |
+
"val_seconds": 7.118,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.8062640825907389,
|
| 53 |
+
"epochs_without_improvement": 1,
|
| 54 |
+
"grad_norm_mean": 0.4491065442562103,
|
| 55 |
+
"grad_norm_max": 1.7096972465515137,
|
| 56 |
+
"clip_fraction": 0.02083333395421505,
|
| 57 |
+
"gpu_peak_gb": 59.729
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.03156406310154125,
|
| 63 |
+
"val_loss": 0.8167936032017072,
|
| 64 |
+
"lr": 0.001,
|
| 65 |
+
"epoch_seconds": 95.59223651885986,
|
| 66 |
+
"train_seconds": 88.34,
|
| 67 |
+
"val_seconds": 7.252,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.8062640825907389,
|
| 72 |
+
"epochs_without_improvement": 2,
|
| 73 |
+
"grad_norm_mean": 0.3927498459815979,
|
| 74 |
+
"grad_norm_max": 1.3694604635238647,
|
| 75 |
+
"clip_fraction": 0.0416666679084301,
|
| 76 |
+
"gpu_peak_gb": 59.729
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.030301747649597626,
|
| 82 |
+
"val_loss": 0.8365312392512957,
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"epoch_seconds": 95.66645789146423,
|
| 85 |
+
"train_seconds": 88.445,
|
| 86 |
+
"val_seconds": 7.222,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.8062640825907389,
|
| 91 |
+
"epochs_without_improvement": 3,
|
| 92 |
+
"grad_norm_mean": 0.33212706446647644,
|
| 93 |
+
"grad_norm_max": 0.8127791881561279,
|
| 94 |
+
"clip_fraction": 0.0,
|
| 95 |
+
"gpu_peak_gb": 59.729
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.030249440964932244,
|
| 101 |
+
"val_loss": 0.8432844579219818,
|
| 102 |
+
"lr": 0.001,
|
| 103 |
+
"epoch_seconds": 95.65944361686707,
|
| 104 |
+
"train_seconds": 88.488,
|
| 105 |
+
"val_seconds": 7.172,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.8062640825907389,
|
| 110 |
+
"epochs_without_improvement": 4,
|
| 111 |
+
"grad_norm_mean": 0.3428223133087158,
|
| 112 |
+
"grad_norm_max": 1.0881145000457764,
|
| 113 |
+
"clip_fraction": 0.010416666977107525,
|
| 114 |
+
"gpu_peak_gb": 59.729
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.029475017605970304,
|
| 120 |
+
"val_loss": 0.8322935303052267,
|
| 121 |
+
"lr": 0.0005,
|
| 122 |
+
"epoch_seconds": 95.57984852790833,
|
| 123 |
+
"train_seconds": 88.354,
|
| 124 |
+
"val_seconds": 7.226,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.8062640825907389,
|
| 129 |
+
"epochs_without_improvement": 5,
|
| 130 |
+
"grad_norm_mean": 0.3077635169029236,
|
| 131 |
+
"grad_norm_max": 1.0377978086471558,
|
| 132 |
+
"clip_fraction": 0.010416666977107525,
|
| 133 |
+
"gpu_peak_gb": 59.729
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.027921032005300123,
|
| 139 |
+
"val_loss": 0.8233420625329018,
|
| 140 |
+
"lr": 0.0005,
|
| 141 |
+
"epoch_seconds": 95.59379386901855,
|
| 142 |
+
"train_seconds": 88.337,
|
| 143 |
+
"val_seconds": 7.257,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.8062640825907389,
|
| 148 |
+
"epochs_without_improvement": 6,
|
| 149 |
+
"grad_norm_mean": 0.17888645827770233,
|
| 150 |
+
"grad_norm_max": 0.8311724662780762,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 59.729
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.02777032503702988,
|
| 158 |
+
"val_loss": 0.823971281449,
|
| 159 |
+
"lr": 0.0005,
|
| 160 |
+
"epoch_seconds": 95.75107383728027,
|
| 161 |
+
"train_seconds": 88.376,
|
| 162 |
+
"val_seconds": 7.375,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.8062640825907389,
|
| 167 |
+
"epochs_without_improvement": 7,
|
| 168 |
+
"grad_norm_mean": 0.193719744682312,
|
| 169 |
+
"grad_norm_max": 0.7943165302276611,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 59.729
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.02729383431142196,
|
| 177 |
+
"val_loss": 0.8238632256786028,
|
| 178 |
+
"lr": 0.0005,
|
| 179 |
+
"epoch_seconds": 95.54115271568298,
|
| 180 |
+
"train_seconds": 88.252,
|
| 181 |
+
"val_seconds": 7.289,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.8062640825907389,
|
| 186 |
+
"epochs_without_improvement": 8,
|
| 187 |
+
"grad_norm_mean": 0.15073560178279877,
|
| 188 |
+
"grad_norm_max": 0.5930942296981812,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 59.729
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.027190778346266598,
|
| 196 |
+
"val_loss": 0.833588071167469,
|
| 197 |
+
"lr": 0.0005,
|
| 198 |
+
"epoch_seconds": 95.51814985275269,
|
| 199 |
+
"train_seconds": 88.332,
|
| 200 |
+
"val_seconds": 7.186,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.8062640825907389,
|
| 205 |
+
"epochs_without_improvement": 9,
|
| 206 |
+
"grad_norm_mean": 0.16863898932933807,
|
| 207 |
+
"grad_norm_max": 0.5619120597839355,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 59.729
|
| 210 |
+
}
|
| 211 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/logs/transformer_asa_3d.log
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:27:42 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:27:42 - root - INFO - Model parameters: 910,278
|
| 3 |
+
2026-09-03 03:27:42 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 03:29:19 - root - INFO - Epoch 0 [train] - train_loss: 0.078043 | val_loss: 0.806264 | lr: 0.001000 | epoch_seconds: 97.520493 | train_seconds: 90.220000 | val_seconds: 7.300000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 0.000000 | grad_norm_mean: 1.715590 | grad_norm_max: 23.972929 | clip_fraction: 0.541667 | gpu_peak_gb: 59.729000
|
| 5 |
+
2026-09-03 03:30:55 - root - INFO - Epoch 1 [train] - train_loss: 0.036394 | val_loss: 0.814743 | lr: 0.001000 | epoch_seconds: 95.364358 | train_seconds: 88.243000 | val_seconds: 7.121000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.574100 | grad_norm_max: 2.131017 | clip_fraction: 0.104167 | gpu_peak_gb: 59.729000
|
| 6 |
+
2026-09-03 03:32:30 - root - INFO - Epoch 2 [train] - train_loss: 0.033346 | val_loss: 0.808985 | lr: 0.001000 | epoch_seconds: 95.432920 | train_seconds: 88.315000 | val_seconds: 7.118000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.449107 | grad_norm_max: 1.709697 | clip_fraction: 0.020833 | gpu_peak_gb: 59.729000
|
| 7 |
+
2026-09-03 03:34:06 - root - INFO - Epoch 3 [train] - train_loss: 0.031564 | val_loss: 0.816794 | lr: 0.001000 | epoch_seconds: 95.592237 | train_seconds: 88.340000 | val_seconds: 7.252000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.392750 | grad_norm_max: 1.369460 | clip_fraction: 0.041667 | gpu_peak_gb: 59.729000
|
| 8 |
+
2026-09-03 03:35:41 - root - INFO - Epoch 4 [train] - train_loss: 0.030302 | val_loss: 0.836531 | lr: 0.001000 | epoch_seconds: 95.666458 | train_seconds: 88.445000 | val_seconds: 7.222000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.332127 | grad_norm_max: 0.812779 | clip_fraction: 0.000000 | gpu_peak_gb: 59.729000
|
| 9 |
+
2026-09-03 03:37:17 - root - INFO - Epoch 5 [train] - train_loss: 0.030249 | val_loss: 0.843284 | lr: 0.001000 | epoch_seconds: 95.659444 | train_seconds: 88.488000 | val_seconds: 7.172000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.342822 | grad_norm_max: 1.088115 | clip_fraction: 0.010417 | gpu_peak_gb: 59.729000
|
| 10 |
+
2026-09-03 03:38:53 - root - INFO - Epoch 6 [train] - train_loss: 0.029475 | val_loss: 0.832294 | lr: 0.000500 | epoch_seconds: 95.579849 | train_seconds: 88.354000 | val_seconds: 7.226000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.307764 | grad_norm_max: 1.037798 | clip_fraction: 0.010417 | gpu_peak_gb: 59.729000
|
| 11 |
+
2026-09-03 03:40:28 - root - INFO - Epoch 7 [train] - train_loss: 0.027921 | val_loss: 0.823342 | lr: 0.000500 | epoch_seconds: 95.593794 | train_seconds: 88.337000 | val_seconds: 7.257000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.178886 | grad_norm_max: 0.831172 | clip_fraction: 0.000000 | gpu_peak_gb: 59.729000
|
| 12 |
+
2026-09-03 03:42:04 - root - INFO - Epoch 8 [train] - train_loss: 0.027770 | val_loss: 0.823971 | lr: 0.000500 | epoch_seconds: 95.751074 | train_seconds: 88.376000 | val_seconds: 7.375000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.193720 | grad_norm_max: 0.794317 | clip_fraction: 0.000000 | gpu_peak_gb: 59.729000
|
| 13 |
+
2026-09-03 03:43:39 - root - INFO - Epoch 9 [train] - train_loss: 0.027294 | val_loss: 0.823863 | lr: 0.000500 | epoch_seconds: 95.541153 | train_seconds: 88.252000 | val_seconds: 7.289000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.150736 | grad_norm_max: 0.593094 | clip_fraction: 0.000000 | gpu_peak_gb: 59.729000
|
| 14 |
+
2026-09-03 03:45:15 - root - INFO - Epoch 10 [train] - train_loss: 0.027191 | val_loss: 0.833588 | lr: 0.000500 | epoch_seconds: 95.518150 | train_seconds: 88.332000 | val_seconds: 7.186000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.806264 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.168639 | grad_norm_max: 0.561912 | clip_fraction: 0.000000 | gpu_peak_gb: 59.729000
|
| 15 |
+
2026-09-03 03:45:15 - root - INFO - Early stopping triggered at epoch 10
|
| 16 |
+
2026-09-03 03:45:15 - root - INFO - Training completed in 1053.24 seconds
|
| 17 |
+
2026-09-03 03:45:15 - root - INFO - Realized optimizer steps: 1056 (96/epoch x 11 epochs) at effective batch 30087 observations
|
| 18 |
+
2026-09-03 03:45:17 - root - INFO - Cost [measured]: 9581.69 TFLOP total (fwd+bwd); 871.06 TFLOP to best (epoch 1)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_3d_embeddings_s619/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_asa_3d_embeddings_s619",
|
| 3 |
+
"fingerprint": "1df50d065a1c4c4ba6dcc4454ae1e937a214c40cda2b8458d9b44b8f07f52206",
|
| 4 |
+
"checkpoint_sha256": "de567d8449e77aec06c47fc7b095c968148ae43bd6edf94c98d8b0759796e7c3"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:de36f7beb71d068b47773419eb7f6104d385b5ba29ad43c0d1d281762fbbc283
|
| 3 |
+
size 4914435
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 9454223197440,
|
| 7 |
+
"flops_per_observation": 314229507.67574036,
|
| 8 |
+
"epochs_run": 25,
|
| 9 |
+
"epochs_to_best": 15,
|
| 10 |
+
"optimizer_steps_total": 2400,
|
| 11 |
+
"optimizer_steps_to_best": 1440,
|
| 12 |
+
"observations_total": 72208800,
|
| 13 |
+
"observations_to_best": 43325280,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 2.2690135673856e+16,
|
| 16 |
+
"train_flops_to_best": 1.36140814043136e+16,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "asa_4d",
|
| 20 |
+
"seed": 619
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/logs/metrics.json
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.12177865583604823,
|
| 6 |
+
"val_loss": 0.9244095881779989,
|
| 7 |
+
"lr": 0.001,
|
| 8 |
+
"epoch_seconds": 104.24988508224487,
|
| 9 |
+
"train_seconds": 96.47,
|
| 10 |
+
"val_seconds": 7.78,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.9244095881779989,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 2.5141448974609375,
|
| 17 |
+
"grad_norm_max": 38.96526336669922,
|
| 18 |
+
"clip_fraction": 0.59375,
|
| 19 |
+
"gpu_peak_gb": 63.209
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.04640512079155693,
|
| 25 |
+
"val_loss": 0.8647230168183645,
|
| 26 |
+
"lr": 0.001,
|
| 27 |
+
"epoch_seconds": 101.91540122032166,
|
| 28 |
+
"train_seconds": 94.152,
|
| 29 |
+
"val_seconds": 7.764,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.8647230168183645,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.6774128675460815,
|
| 36 |
+
"grad_norm_max": 2.1699321269989014,
|
| 37 |
+
"clip_fraction": 0.2395833432674408,
|
| 38 |
+
"gpu_peak_gb": 63.209
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.03820654115406796,
|
| 44 |
+
"val_loss": 0.8200776303807894,
|
| 45 |
+
"lr": 0.001,
|
| 46 |
+
"epoch_seconds": 101.96345973014832,
|
| 47 |
+
"train_seconds": 94.179,
|
| 48 |
+
"val_seconds": 7.784,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.8200776303807894,
|
| 53 |
+
"epochs_without_improvement": 0,
|
| 54 |
+
"grad_norm_mean": 0.4343891739845276,
|
| 55 |
+
"grad_norm_max": 1.4148380756378174,
|
| 56 |
+
"clip_fraction": 0.03125,
|
| 57 |
+
"gpu_peak_gb": 63.209
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.034086261255045734,
|
| 63 |
+
"val_loss": 0.8360788822174072,
|
| 64 |
+
"lr": 0.001,
|
| 65 |
+
"epoch_seconds": 101.79995250701904,
|
| 66 |
+
"train_seconds": 94.118,
|
| 67 |
+
"val_seconds": 7.682,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.8200776303807894,
|
| 72 |
+
"epochs_without_improvement": 0,
|
| 73 |
+
"grad_norm_mean": 0.3649986684322357,
|
| 74 |
+
"grad_norm_max": 1.2578015327453613,
|
| 75 |
+
"clip_fraction": 0.02083333395421505,
|
| 76 |
+
"gpu_peak_gb": 63.209
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.0317471234787566,
|
| 82 |
+
"val_loss": 0.8440884773929914,
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"epoch_seconds": 101.76579260826111,
|
| 85 |
+
"train_seconds": 93.991,
|
| 86 |
+
"val_seconds": 7.774,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.8200776303807894,
|
| 91 |
+
"epochs_without_improvement": 1,
|
| 92 |
+
"grad_norm_mean": 0.2785022258758545,
|
| 93 |
+
"grad_norm_max": 0.7020371556282043,
|
| 94 |
+
"clip_fraction": 0.0,
|
| 95 |
+
"gpu_peak_gb": 63.209
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.031266435786771275,
|
| 101 |
+
"val_loss": 0.844168725113074,
|
| 102 |
+
"lr": 0.001,
|
| 103 |
+
"epoch_seconds": 101.71682095527649,
|
| 104 |
+
"train_seconds": 94.046,
|
| 105 |
+
"val_seconds": 7.671,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.8200776303807894,
|
| 110 |
+
"epochs_without_improvement": 2,
|
| 111 |
+
"grad_norm_mean": 0.2739657163619995,
|
| 112 |
+
"grad_norm_max": 0.6159876585006714,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 63.209
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.031492950627580285,
|
| 120 |
+
"val_loss": 0.8438127736250559,
|
| 121 |
+
"lr": 0.001,
|
| 122 |
+
"epoch_seconds": 101.77490019798279,
|
| 123 |
+
"train_seconds": 93.96,
|
| 124 |
+
"val_seconds": 7.815,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.8200776303807894,
|
| 129 |
+
"epochs_without_improvement": 3,
|
| 130 |
+
"grad_norm_mean": 0.3247450292110443,
|
| 131 |
+
"grad_norm_max": 0.8616554737091064,
|
| 132 |
+
"clip_fraction": 0.0,
|
| 133 |
+
"gpu_peak_gb": 63.209
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.029026798496488482,
|
| 139 |
+
"val_loss": 0.8343626310427984,
|
| 140 |
+
"lr": 0.001,
|
| 141 |
+
"epoch_seconds": 101.86091089248657,
|
| 142 |
+
"train_seconds": 94.029,
|
| 143 |
+
"val_seconds": 7.832,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.8200776303807894,
|
| 148 |
+
"epochs_without_improvement": 4,
|
| 149 |
+
"grad_norm_mean": 0.20740735530853271,
|
| 150 |
+
"grad_norm_max": 0.7999801635742188,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 63.209
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.029033780951673787,
|
| 158 |
+
"val_loss": 0.8142769262194633,
|
| 159 |
+
"lr": 0.001,
|
| 160 |
+
"epoch_seconds": 102.17947769165039,
|
| 161 |
+
"train_seconds": 94.098,
|
| 162 |
+
"val_seconds": 8.081,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.8142769262194633,
|
| 167 |
+
"epochs_without_improvement": 5,
|
| 168 |
+
"grad_norm_mean": 0.23117300868034363,
|
| 169 |
+
"grad_norm_max": 0.778771698474884,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 63.209
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.028276651923079044,
|
| 177 |
+
"val_loss": 0.817805290222168,
|
| 178 |
+
"lr": 0.001,
|
| 179 |
+
"epoch_seconds": 102.2635703086853,
|
| 180 |
+
"train_seconds": 94.099,
|
| 181 |
+
"val_seconds": 8.164,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.8142769262194633,
|
| 186 |
+
"epochs_without_improvement": 0,
|
| 187 |
+
"grad_norm_mean": 0.22509202361106873,
|
| 188 |
+
"grad_norm_max": 0.9112066626548767,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 63.209
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.027396563809209813,
|
| 196 |
+
"val_loss": 0.8234609241286913,
|
| 197 |
+
"lr": 0.001,
|
| 198 |
+
"epoch_seconds": 102.10460996627808,
|
| 199 |
+
"train_seconds": 94.225,
|
| 200 |
+
"val_seconds": 7.88,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.8142769262194633,
|
| 205 |
+
"epochs_without_improvement": 1,
|
| 206 |
+
"grad_norm_mean": 0.16831962764263153,
|
| 207 |
+
"grad_norm_max": 0.5869952440261841,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 63.209
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"epoch": 11,
|
| 213 |
+
"phase": "train",
|
| 214 |
+
"train_loss": 0.027647608891129494,
|
| 215 |
+
"val_loss": 0.8002436757087708,
|
| 216 |
+
"lr": 0.001,
|
| 217 |
+
"epoch_seconds": 101.93613839149475,
|
| 218 |
+
"train_seconds": 94.044,
|
| 219 |
+
"val_seconds": 7.892,
|
| 220 |
+
"steps": 96,
|
| 221 |
+
"total_steps": 1152,
|
| 222 |
+
"observations": 34660224,
|
| 223 |
+
"best_val_loss": 0.8002436757087708,
|
| 224 |
+
"epochs_without_improvement": 2,
|
| 225 |
+
"grad_norm_mean": 0.20128285884857178,
|
| 226 |
+
"grad_norm_max": 0.6246017217636108,
|
| 227 |
+
"clip_fraction": 0.0,
|
| 228 |
+
"gpu_peak_gb": 63.209
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"epoch": 12,
|
| 232 |
+
"phase": "train",
|
| 233 |
+
"train_loss": 0.026898861591083307,
|
| 234 |
+
"val_loss": 0.8034286672870318,
|
| 235 |
+
"lr": 0.001,
|
| 236 |
+
"epoch_seconds": 101.90797400474548,
|
| 237 |
+
"train_seconds": 94.047,
|
| 238 |
+
"val_seconds": 7.861,
|
| 239 |
+
"steps": 96,
|
| 240 |
+
"total_steps": 1248,
|
| 241 |
+
"observations": 37548576,
|
| 242 |
+
"best_val_loss": 0.8002436757087708,
|
| 243 |
+
"epochs_without_improvement": 0,
|
| 244 |
+
"grad_norm_mean": 0.1614457070827484,
|
| 245 |
+
"grad_norm_max": 0.5405094027519226,
|
| 246 |
+
"clip_fraction": 0.0,
|
| 247 |
+
"gpu_peak_gb": 63.209
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"epoch": 13,
|
| 251 |
+
"phase": "train",
|
| 252 |
+
"train_loss": 0.02759501033384974,
|
| 253 |
+
"val_loss": 0.8055128653844198,
|
| 254 |
+
"lr": 0.001,
|
| 255 |
+
"epoch_seconds": 101.9175534248352,
|
| 256 |
+
"train_seconds": 94.025,
|
| 257 |
+
"val_seconds": 7.892,
|
| 258 |
+
"steps": 96,
|
| 259 |
+
"total_steps": 1344,
|
| 260 |
+
"observations": 40436928,
|
| 261 |
+
"best_val_loss": 0.8002436757087708,
|
| 262 |
+
"epochs_without_improvement": 1,
|
| 263 |
+
"grad_norm_mean": 0.2060072422027588,
|
| 264 |
+
"grad_norm_max": 0.5987624526023865,
|
| 265 |
+
"clip_fraction": 0.0,
|
| 266 |
+
"gpu_peak_gb": 63.209
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"epoch": 14,
|
| 270 |
+
"phase": "train",
|
| 271 |
+
"train_loss": 0.02642867520141105,
|
| 272 |
+
"val_loss": 0.7924586782852808,
|
| 273 |
+
"lr": 0.001,
|
| 274 |
+
"epoch_seconds": 101.85883355140686,
|
| 275 |
+
"train_seconds": 94.074,
|
| 276 |
+
"val_seconds": 7.785,
|
| 277 |
+
"steps": 96,
|
| 278 |
+
"total_steps": 1440,
|
| 279 |
+
"observations": 43325280,
|
| 280 |
+
"best_val_loss": 0.7924586782852808,
|
| 281 |
+
"epochs_without_improvement": 2,
|
| 282 |
+
"grad_norm_mean": 0.13279025256633759,
|
| 283 |
+
"grad_norm_max": 0.9188597202301025,
|
| 284 |
+
"clip_fraction": 0.0,
|
| 285 |
+
"gpu_peak_gb": 63.209
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"epoch": 15,
|
| 289 |
+
"phase": "train",
|
| 290 |
+
"train_loss": 0.026696813156983506,
|
| 291 |
+
"val_loss": 0.8209474335114161,
|
| 292 |
+
"lr": 0.001,
|
| 293 |
+
"epoch_seconds": 101.82543778419495,
|
| 294 |
+
"train_seconds": 93.998,
|
| 295 |
+
"val_seconds": 7.827,
|
| 296 |
+
"steps": 96,
|
| 297 |
+
"total_steps": 1536,
|
| 298 |
+
"observations": 46213632,
|
| 299 |
+
"best_val_loss": 0.7924586782852808,
|
| 300 |
+
"epochs_without_improvement": 0,
|
| 301 |
+
"grad_norm_mean": 0.1808527410030365,
|
| 302 |
+
"grad_norm_max": 0.42495983839035034,
|
| 303 |
+
"clip_fraction": 0.0,
|
| 304 |
+
"gpu_peak_gb": 63.209
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"epoch": 16,
|
| 308 |
+
"phase": "train",
|
| 309 |
+
"train_loss": 0.026225996339538444,
|
| 310 |
+
"val_loss": 0.8413715362548828,
|
| 311 |
+
"lr": 0.001,
|
| 312 |
+
"epoch_seconds": 101.92606234550476,
|
| 313 |
+
"train_seconds": 94.08,
|
| 314 |
+
"val_seconds": 7.846,
|
| 315 |
+
"steps": 96,
|
| 316 |
+
"total_steps": 1632,
|
| 317 |
+
"observations": 49101984,
|
| 318 |
+
"best_val_loss": 0.7924586782852808,
|
| 319 |
+
"epochs_without_improvement": 1,
|
| 320 |
+
"grad_norm_mean": 0.1360030174255371,
|
| 321 |
+
"grad_norm_max": 0.42238649725914,
|
| 322 |
+
"clip_fraction": 0.0,
|
| 323 |
+
"gpu_peak_gb": 63.209
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"epoch": 17,
|
| 327 |
+
"phase": "train",
|
| 328 |
+
"train_loss": 0.026064549359337736,
|
| 329 |
+
"val_loss": 0.8658649722735087,
|
| 330 |
+
"lr": 0.001,
|
| 331 |
+
"epoch_seconds": 101.73835039138794,
|
| 332 |
+
"train_seconds": 94.011,
|
| 333 |
+
"val_seconds": 7.727,
|
| 334 |
+
"steps": 96,
|
| 335 |
+
"total_steps": 1728,
|
| 336 |
+
"observations": 51990336,
|
| 337 |
+
"best_val_loss": 0.7924586782852808,
|
| 338 |
+
"epochs_without_improvement": 2,
|
| 339 |
+
"grad_norm_mean": 0.13067740201950073,
|
| 340 |
+
"grad_norm_max": 0.5284085869789124,
|
| 341 |
+
"clip_fraction": 0.0,
|
| 342 |
+
"gpu_peak_gb": 63.209
|
| 343 |
+
},
|
| 344 |
+
{
|
| 345 |
+
"epoch": 18,
|
| 346 |
+
"phase": "train",
|
| 347 |
+
"train_loss": 0.026751265511848032,
|
| 348 |
+
"val_loss": 0.8463066220283508,
|
| 349 |
+
"lr": 0.001,
|
| 350 |
+
"epoch_seconds": 101.71269631385803,
|
| 351 |
+
"train_seconds": 93.991,
|
| 352 |
+
"val_seconds": 7.721,
|
| 353 |
+
"steps": 96,
|
| 354 |
+
"total_steps": 1824,
|
| 355 |
+
"observations": 54878688,
|
| 356 |
+
"best_val_loss": 0.7924586782852808,
|
| 357 |
+
"epochs_without_improvement": 3,
|
| 358 |
+
"grad_norm_mean": 0.18759042024612427,
|
| 359 |
+
"grad_norm_max": 0.4980805814266205,
|
| 360 |
+
"clip_fraction": 0.0,
|
| 361 |
+
"gpu_peak_gb": 63.209
|
| 362 |
+
},
|
| 363 |
+
{
|
| 364 |
+
"epoch": 19,
|
| 365 |
+
"phase": "train",
|
| 366 |
+
"train_loss": 0.025846837321296334,
|
| 367 |
+
"val_loss": 0.9081577956676483,
|
| 368 |
+
"lr": 0.001,
|
| 369 |
+
"epoch_seconds": 101.94386029243469,
|
| 370 |
+
"train_seconds": 94.067,
|
| 371 |
+
"val_seconds": 7.877,
|
| 372 |
+
"steps": 96,
|
| 373 |
+
"total_steps": 1920,
|
| 374 |
+
"observations": 57767040,
|
| 375 |
+
"best_val_loss": 0.7924586782852808,
|
| 376 |
+
"epochs_without_improvement": 4,
|
| 377 |
+
"grad_norm_mean": 0.12281805276870728,
|
| 378 |
+
"grad_norm_max": 0.4824073910713196,
|
| 379 |
+
"clip_fraction": 0.0,
|
| 380 |
+
"gpu_peak_gb": 63.209
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"epoch": 20,
|
| 384 |
+
"phase": "train",
|
| 385 |
+
"train_loss": 0.025534685041444998,
|
| 386 |
+
"val_loss": 0.8823175951838493,
|
| 387 |
+
"lr": 0.0005,
|
| 388 |
+
"epoch_seconds": 101.79942488670349,
|
| 389 |
+
"train_seconds": 93.963,
|
| 390 |
+
"val_seconds": 7.836,
|
| 391 |
+
"steps": 96,
|
| 392 |
+
"total_steps": 2016,
|
| 393 |
+
"observations": 60655392,
|
| 394 |
+
"best_val_loss": 0.7924586782852808,
|
| 395 |
+
"epochs_without_improvement": 5,
|
| 396 |
+
"grad_norm_mean": 0.10888338088989258,
|
| 397 |
+
"grad_norm_max": 0.4773053824901581,
|
| 398 |
+
"clip_fraction": 0.0,
|
| 399 |
+
"gpu_peak_gb": 63.209
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"epoch": 21,
|
| 403 |
+
"phase": "train",
|
| 404 |
+
"train_loss": 0.025431178665409487,
|
| 405 |
+
"val_loss": 0.8732851097981135,
|
| 406 |
+
"lr": 0.0005,
|
| 407 |
+
"epoch_seconds": 101.75363874435425,
|
| 408 |
+
"train_seconds": 94.044,
|
| 409 |
+
"val_seconds": 7.71,
|
| 410 |
+
"steps": 96,
|
| 411 |
+
"total_steps": 2112,
|
| 412 |
+
"observations": 63543744,
|
| 413 |
+
"best_val_loss": 0.7924586782852808,
|
| 414 |
+
"epochs_without_improvement": 6,
|
| 415 |
+
"grad_norm_mean": 0.10395544767379761,
|
| 416 |
+
"grad_norm_max": 0.5154268741607666,
|
| 417 |
+
"clip_fraction": 0.0,
|
| 418 |
+
"gpu_peak_gb": 63.209
|
| 419 |
+
},
|
| 420 |
+
{
|
| 421 |
+
"epoch": 22,
|
| 422 |
+
"phase": "train",
|
| 423 |
+
"train_loss": 0.02530760100732247,
|
| 424 |
+
"val_loss": 0.8812726686398188,
|
| 425 |
+
"lr": 0.0005,
|
| 426 |
+
"epoch_seconds": 102.16074252128601,
|
| 427 |
+
"train_seconds": 93.962,
|
| 428 |
+
"val_seconds": 8.199,
|
| 429 |
+
"steps": 96,
|
| 430 |
+
"total_steps": 2208,
|
| 431 |
+
"observations": 66432096,
|
| 432 |
+
"best_val_loss": 0.7924586782852808,
|
| 433 |
+
"epochs_without_improvement": 7,
|
| 434 |
+
"grad_norm_mean": 0.10218356549739838,
|
| 435 |
+
"grad_norm_max": 0.39187321066856384,
|
| 436 |
+
"clip_fraction": 0.0,
|
| 437 |
+
"gpu_peak_gb": 63.209
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"epoch": 23,
|
| 441 |
+
"phase": "train",
|
| 442 |
+
"train_loss": 0.025191764754708856,
|
| 443 |
+
"val_loss": 0.8751332114140192,
|
| 444 |
+
"lr": 0.0005,
|
| 445 |
+
"epoch_seconds": 102.04192900657654,
|
| 446 |
+
"train_seconds": 94.079,
|
| 447 |
+
"val_seconds": 7.963,
|
| 448 |
+
"steps": 96,
|
| 449 |
+
"total_steps": 2304,
|
| 450 |
+
"observations": 69320448,
|
| 451 |
+
"best_val_loss": 0.7924586782852808,
|
| 452 |
+
"epochs_without_improvement": 8,
|
| 453 |
+
"grad_norm_mean": 0.09179534018039703,
|
| 454 |
+
"grad_norm_max": 0.5089642405509949,
|
| 455 |
+
"clip_fraction": 0.0,
|
| 456 |
+
"gpu_peak_gb": 63.209
|
| 457 |
+
},
|
| 458 |
+
{
|
| 459 |
+
"epoch": 24,
|
| 460 |
+
"phase": "train",
|
| 461 |
+
"train_loss": 0.025226510032856215,
|
| 462 |
+
"val_loss": 0.8829309269785881,
|
| 463 |
+
"lr": 0.0005,
|
| 464 |
+
"epoch_seconds": 102.11145424842834,
|
| 465 |
+
"train_seconds": 94.124,
|
| 466 |
+
"val_seconds": 7.988,
|
| 467 |
+
"steps": 96,
|
| 468 |
+
"total_steps": 2400,
|
| 469 |
+
"observations": 72208800,
|
| 470 |
+
"best_val_loss": 0.7924586782852808,
|
| 471 |
+
"epochs_without_improvement": 9,
|
| 472 |
+
"grad_norm_mean": 0.10891138017177582,
|
| 473 |
+
"grad_norm_max": 0.33056119084358215,
|
| 474 |
+
"clip_fraction": 0.0,
|
| 475 |
+
"gpu_peak_gb": 63.209
|
| 476 |
+
}
|
| 477 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/logs/transformer_asa_4d.log
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:31:24 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:31:24 - root - INFO - Model parameters: 959,170
|
| 3 |
+
2026-09-03 03:31:24 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 03:33:06 - root - INFO - Epoch 0 [train] - train_loss: 0.121779 | val_loss: 0.924410 | lr: 0.001000 | epoch_seconds: 102.571290 | train_seconds: 95.066000 | val_seconds: 7.505000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.924410 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.514145 | grad_norm_max: 38.965263 | clip_fraction: 0.593750 | gpu_peak_gb: 63.209000
|
| 5 |
+
2026-09-03 03:34:47 - root - INFO - Epoch 1 [train] - train_loss: 0.046405 | val_loss: 0.864723 | lr: 0.001000 | epoch_seconds: 100.787041 | train_seconds: 93.306000 | val_seconds: 7.481000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.864723 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.677413 | grad_norm_max: 2.169932 | clip_fraction: 0.239583 | gpu_peak_gb: 63.209000
|
| 6 |
+
2026-09-03 03:36:28 - root - INFO - Epoch 2 [train] - train_loss: 0.038207 | val_loss: 0.820078 | lr: 0.001000 | epoch_seconds: 100.802838 | train_seconds: 93.188000 | val_seconds: 7.615000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.434389 | grad_norm_max: 1.414838 | clip_fraction: 0.031250 | gpu_peak_gb: 63.209000
|
| 7 |
+
2026-09-03 03:38:09 - root - INFO - Epoch 3 [train] - train_loss: 0.034086 | val_loss: 0.836079 | lr: 0.001000 | epoch_seconds: 100.839493 | train_seconds: 93.312000 | val_seconds: 7.527000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.364999 | grad_norm_max: 1.257802 | clip_fraction: 0.020833 | gpu_peak_gb: 63.209000
|
| 8 |
+
2026-09-03 03:39:50 - root - INFO - Epoch 4 [train] - train_loss: 0.031747 | val_loss: 0.844088 | lr: 0.001000 | epoch_seconds: 100.783032 | train_seconds: 93.220000 | val_seconds: 7.563000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.278502 | grad_norm_max: 0.702037 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 9 |
+
2026-09-03 03:41:31 - root - INFO - Epoch 5 [train] - train_loss: 0.031266 | val_loss: 0.844169 | lr: 0.001000 | epoch_seconds: 100.876660 | train_seconds: 93.241000 | val_seconds: 7.636000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.273966 | grad_norm_max: 0.615988 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 10 |
+
2026-09-03 03:43:12 - root - INFO - Epoch 6 [train] - train_loss: 0.031493 | val_loss: 0.843813 | lr: 0.001000 | epoch_seconds: 100.952345 | train_seconds: 93.287000 | val_seconds: 7.665000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.324745 | grad_norm_max: 0.861655 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 11 |
+
2026-09-03 03:44:52 - root - INFO - Epoch 7 [train] - train_loss: 0.029027 | val_loss: 0.834363 | lr: 0.001000 | epoch_seconds: 100.722483 | train_seconds: 93.205000 | val_seconds: 7.517000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.207407 | grad_norm_max: 0.799980 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 12 |
+
2026-09-03 03:46:33 - root - INFO - Epoch 8 [train] - train_loss: 0.029034 | val_loss: 0.814277 | lr: 0.001000 | epoch_seconds: 100.684460 | train_seconds: 93.180000 | val_seconds: 7.505000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.231173 | grad_norm_max: 0.778772 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 13 |
+
2026-09-03 03:48:14 - root - INFO - Epoch 9 [train] - train_loss: 0.028277 | val_loss: 0.817805 | lr: 0.001000 | epoch_seconds: 100.882140 | train_seconds: 93.280000 | val_seconds: 7.602000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.225092 | grad_norm_max: 0.911207 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 14 |
+
2026-09-03 03:49:55 - root - INFO - Epoch 10 [train] - train_loss: 0.027397 | val_loss: 0.823461 | lr: 0.001000 | epoch_seconds: 100.785206 | train_seconds: 93.167000 | val_seconds: 7.618000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.168320 | grad_norm_max: 0.586995 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 15 |
+
2026-09-03 03:51:35 - root - INFO - Epoch 11 [train] - train_loss: 0.027648 | val_loss: 0.800244 | lr: 0.001000 | epoch_seconds: 100.750383 | train_seconds: 93.113000 | val_seconds: 7.637000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.201283 | grad_norm_max: 0.624602 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 16 |
+
2026-09-03 03:53:16 - root - INFO - Epoch 12 [train] - train_loss: 0.026899 | val_loss: 0.803429 | lr: 0.001000 | epoch_seconds: 100.813718 | train_seconds: 93.286000 | val_seconds: 7.528000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.161446 | grad_norm_max: 0.540509 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 17 |
+
2026-09-03 03:54:57 - root - INFO - Epoch 13 [train] - train_loss: 0.027595 | val_loss: 0.805513 | lr: 0.001000 | epoch_seconds: 100.632382 | train_seconds: 93.133000 | val_seconds: 7.499000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.206007 | grad_norm_max: 0.598762 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 18 |
+
2026-09-03 03:56:38 - root - INFO - Epoch 14 [train] - train_loss: 0.026429 | val_loss: 0.792459 | lr: 0.001000 | epoch_seconds: 100.915519 | train_seconds: 93.297000 | val_seconds: 7.619000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.132790 | grad_norm_max: 0.918860 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 19 |
+
2026-09-03 03:58:19 - root - INFO - Epoch 15 [train] - train_loss: 0.026697 | val_loss: 0.820947 | lr: 0.001000 | epoch_seconds: 101.068870 | train_seconds: 93.218000 | val_seconds: 7.851000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.180853 | grad_norm_max: 0.424960 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 20 |
+
2026-09-03 04:00:00 - root - INFO - Epoch 16 [train] - train_loss: 0.026226 | val_loss: 0.841372 | lr: 0.001000 | epoch_seconds: 100.692055 | train_seconds: 93.182000 | val_seconds: 7.510000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.136003 | grad_norm_max: 0.422386 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 21 |
+
2026-09-03 04:01:41 - root - INFO - Epoch 17 [train] - train_loss: 0.026065 | val_loss: 0.865865 | lr: 0.001000 | epoch_seconds: 101.303978 | train_seconds: 93.592000 | val_seconds: 7.712000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.130677 | grad_norm_max: 0.528409 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 22 |
+
2026-09-03 04:03:22 - root - INFO - Epoch 18 [train] - train_loss: 0.026751 | val_loss: 0.846307 | lr: 0.001000 | epoch_seconds: 100.769218 | train_seconds: 93.153000 | val_seconds: 7.616000 | steps: 96.000000 | total_steps: 1824.000000 | observations: 54878688.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.187590 | grad_norm_max: 0.498081 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 23 |
+
2026-09-03 04:05:02 - root - INFO - Epoch 19 [train] - train_loss: 0.025847 | val_loss: 0.908158 | lr: 0.001000 | epoch_seconds: 100.803281 | train_seconds: 93.196000 | val_seconds: 7.607000 | steps: 96.000000 | total_steps: 1920.000000 | observations: 57767040.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.122818 | grad_norm_max: 0.482407 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 24 |
+
2026-09-03 04:06:43 - root - INFO - Epoch 20 [train] - train_loss: 0.025535 | val_loss: 0.882318 | lr: 0.000500 | epoch_seconds: 101.026867 | train_seconds: 93.309000 | val_seconds: 7.718000 | steps: 96.000000 | total_steps: 2016.000000 | observations: 60655392.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.108883 | grad_norm_max: 0.477305 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 25 |
+
2026-09-03 04:08:24 - root - INFO - Epoch 21 [train] - train_loss: 0.025431 | val_loss: 0.873285 | lr: 0.000500 | epoch_seconds: 100.805722 | train_seconds: 93.218000 | val_seconds: 7.587000 | steps: 96.000000 | total_steps: 2112.000000 | observations: 63543744.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.103955 | grad_norm_max: 0.515427 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 26 |
+
2026-09-03 04:10:05 - root - INFO - Epoch 22 [train] - train_loss: 0.025308 | val_loss: 0.881273 | lr: 0.000500 | epoch_seconds: 100.987261 | train_seconds: 93.193000 | val_seconds: 7.794000 | steps: 96.000000 | total_steps: 2208.000000 | observations: 66432096.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.102184 | grad_norm_max: 0.391873 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 27 |
+
2026-09-03 04:11:46 - root - INFO - Epoch 23 [train] - train_loss: 0.025192 | val_loss: 0.875133 | lr: 0.000500 | epoch_seconds: 100.864015 | train_seconds: 93.183000 | val_seconds: 7.681000 | steps: 96.000000 | total_steps: 2304.000000 | observations: 69320448.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.091795 | grad_norm_max: 0.508964 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 28 |
+
2026-09-03 04:13:27 - root - INFO - Epoch 24 [train] - train_loss: 0.025227 | val_loss: 0.882931 | lr: 0.000500 | epoch_seconds: 100.801588 | train_seconds: 93.250000 | val_seconds: 7.551000 | steps: 96.000000 | total_steps: 2400.000000 | observations: 72208800.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.108911 | grad_norm_max: 0.330561 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 29 |
+
2026-09-03 04:13:27 - root - INFO - Early stopping triggered at epoch 24
|
| 30 |
+
2026-09-03 04:13:27 - root - INFO - Training completed in 2523.00 seconds
|
| 31 |
+
2026-09-03 04:13:27 - root - INFO - Realized optimizer steps: 2400 (96/epoch x 25 epochs) at effective batch 30087 observations
|
| 32 |
+
2026-09-03 04:13:29 - root - INFO - Cost [measured]: 22690.14 TFLOP total (fwd+bwd); 13614.08 TFLOP to best (epoch 15)
|
| 33 |
+
2026-09-03 10:46:50 - root - INFO - Trainer initialized on device: cuda
|
| 34 |
+
2026-09-03 10:46:50 - root - INFO - Model parameters: 959,170
|
| 35 |
+
2026-09-03 10:46:50 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 36 |
+
2026-09-03 10:48:35 - root - INFO - Epoch 0 [train] - train_loss: 0.121779 | val_loss: 0.924410 | lr: 0.001000 | epoch_seconds: 104.249885 | train_seconds: 96.470000 | val_seconds: 7.780000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.924410 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.514145 | grad_norm_max: 38.965263 | clip_fraction: 0.593750 | gpu_peak_gb: 63.209000
|
| 37 |
+
2026-09-03 10:50:17 - root - INFO - Epoch 1 [train] - train_loss: 0.046405 | val_loss: 0.864723 | lr: 0.001000 | epoch_seconds: 101.915401 | train_seconds: 94.152000 | val_seconds: 7.764000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.864723 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.677413 | grad_norm_max: 2.169932 | clip_fraction: 0.239583 | gpu_peak_gb: 63.209000
|
| 38 |
+
2026-09-03 10:51:59 - root - INFO - Epoch 2 [train] - train_loss: 0.038207 | val_loss: 0.820078 | lr: 0.001000 | epoch_seconds: 101.963460 | train_seconds: 94.179000 | val_seconds: 7.784000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.434389 | grad_norm_max: 1.414838 | clip_fraction: 0.031250 | gpu_peak_gb: 63.209000
|
| 39 |
+
2026-09-03 10:53:40 - root - INFO - Epoch 3 [train] - train_loss: 0.034086 | val_loss: 0.836079 | lr: 0.001000 | epoch_seconds: 101.799953 | train_seconds: 94.118000 | val_seconds: 7.682000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.364999 | grad_norm_max: 1.257802 | clip_fraction: 0.020833 | gpu_peak_gb: 63.209000
|
| 40 |
+
2026-09-03 10:55:22 - root - INFO - Epoch 4 [train] - train_loss: 0.031747 | val_loss: 0.844088 | lr: 0.001000 | epoch_seconds: 101.765793 | train_seconds: 93.991000 | val_seconds: 7.774000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.278502 | grad_norm_max: 0.702037 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 41 |
+
2026-09-03 10:57:04 - root - INFO - Epoch 5 [train] - train_loss: 0.031266 | val_loss: 0.844169 | lr: 0.001000 | epoch_seconds: 101.716821 | train_seconds: 94.046000 | val_seconds: 7.671000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.273966 | grad_norm_max: 0.615988 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 42 |
+
2026-09-03 10:58:46 - root - INFO - Epoch 6 [train] - train_loss: 0.031493 | val_loss: 0.843813 | lr: 0.001000 | epoch_seconds: 101.774900 | train_seconds: 93.960000 | val_seconds: 7.815000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.324745 | grad_norm_max: 0.861655 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 43 |
+
2026-09-03 11:00:27 - root - INFO - Epoch 7 [train] - train_loss: 0.029027 | val_loss: 0.834363 | lr: 0.001000 | epoch_seconds: 101.860911 | train_seconds: 94.029000 | val_seconds: 7.832000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.820078 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.207407 | grad_norm_max: 0.799980 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 44 |
+
2026-09-03 11:02:10 - root - INFO - Epoch 8 [train] - train_loss: 0.029034 | val_loss: 0.814277 | lr: 0.001000 | epoch_seconds: 102.179478 | train_seconds: 94.098000 | val_seconds: 8.081000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.231173 | grad_norm_max: 0.778772 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 45 |
+
2026-09-03 11:03:52 - root - INFO - Epoch 9 [train] - train_loss: 0.028277 | val_loss: 0.817805 | lr: 0.001000 | epoch_seconds: 102.263570 | train_seconds: 94.099000 | val_seconds: 8.164000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.225092 | grad_norm_max: 0.911207 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 46 |
+
2026-09-03 11:05:34 - root - INFO - Epoch 10 [train] - train_loss: 0.027397 | val_loss: 0.823461 | lr: 0.001000 | epoch_seconds: 102.104610 | train_seconds: 94.225000 | val_seconds: 7.880000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.814277 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.168320 | grad_norm_max: 0.586995 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 47 |
+
2026-09-03 11:07:16 - root - INFO - Epoch 11 [train] - train_loss: 0.027648 | val_loss: 0.800244 | lr: 0.001000 | epoch_seconds: 101.936138 | train_seconds: 94.044000 | val_seconds: 7.892000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.201283 | grad_norm_max: 0.624602 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 48 |
+
2026-09-03 11:08:58 - root - INFO - Epoch 12 [train] - train_loss: 0.026899 | val_loss: 0.803429 | lr: 0.001000 | epoch_seconds: 101.907974 | train_seconds: 94.047000 | val_seconds: 7.861000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.161446 | grad_norm_max: 0.540509 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 49 |
+
2026-09-03 11:10:40 - root - INFO - Epoch 13 [train] - train_loss: 0.027595 | val_loss: 0.805513 | lr: 0.001000 | epoch_seconds: 101.917553 | train_seconds: 94.025000 | val_seconds: 7.892000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.800244 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.206007 | grad_norm_max: 0.598762 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 50 |
+
2026-09-03 11:12:22 - root - INFO - Epoch 14 [train] - train_loss: 0.026429 | val_loss: 0.792459 | lr: 0.001000 | epoch_seconds: 101.858834 | train_seconds: 94.074000 | val_seconds: 7.785000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.132790 | grad_norm_max: 0.918860 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 51 |
+
2026-09-03 11:14:04 - root - INFO - Epoch 15 [train] - train_loss: 0.026697 | val_loss: 0.820947 | lr: 0.001000 | epoch_seconds: 101.825438 | train_seconds: 93.998000 | val_seconds: 7.827000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.180853 | grad_norm_max: 0.424960 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 52 |
+
2026-09-03 11:15:45 - root - INFO - Epoch 16 [train] - train_loss: 0.026226 | val_loss: 0.841372 | lr: 0.001000 | epoch_seconds: 101.926062 | train_seconds: 94.080000 | val_seconds: 7.846000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.136003 | grad_norm_max: 0.422386 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 53 |
+
2026-09-03 11:17:27 - root - INFO - Epoch 17 [train] - train_loss: 0.026065 | val_loss: 0.865865 | lr: 0.001000 | epoch_seconds: 101.738350 | train_seconds: 94.011000 | val_seconds: 7.727000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.130677 | grad_norm_max: 0.528409 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 54 |
+
2026-09-03 11:19:09 - root - INFO - Epoch 18 [train] - train_loss: 0.026751 | val_loss: 0.846307 | lr: 0.001000 | epoch_seconds: 101.712696 | train_seconds: 93.991000 | val_seconds: 7.721000 | steps: 96.000000 | total_steps: 1824.000000 | observations: 54878688.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.187590 | grad_norm_max: 0.498081 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 55 |
+
2026-09-03 11:20:51 - root - INFO - Epoch 19 [train] - train_loss: 0.025847 | val_loss: 0.908158 | lr: 0.001000 | epoch_seconds: 101.943860 | train_seconds: 94.067000 | val_seconds: 7.877000 | steps: 96.000000 | total_steps: 1920.000000 | observations: 57767040.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.122818 | grad_norm_max: 0.482407 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 56 |
+
2026-09-03 11:22:33 - root - INFO - Epoch 20 [train] - train_loss: 0.025535 | val_loss: 0.882318 | lr: 0.000500 | epoch_seconds: 101.799425 | train_seconds: 93.963000 | val_seconds: 7.836000 | steps: 96.000000 | total_steps: 2016.000000 | observations: 60655392.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.108883 | grad_norm_max: 0.477305 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 57 |
+
2026-09-03 11:24:14 - root - INFO - Epoch 21 [train] - train_loss: 0.025431 | val_loss: 0.873285 | lr: 0.000500 | epoch_seconds: 101.753639 | train_seconds: 94.044000 | val_seconds: 7.710000 | steps: 96.000000 | total_steps: 2112.000000 | observations: 63543744.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.103955 | grad_norm_max: 0.515427 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 58 |
+
2026-09-03 11:25:57 - root - INFO - Epoch 22 [train] - train_loss: 0.025308 | val_loss: 0.881273 | lr: 0.000500 | epoch_seconds: 102.160743 | train_seconds: 93.962000 | val_seconds: 8.199000 | steps: 96.000000 | total_steps: 2208.000000 | observations: 66432096.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.102184 | grad_norm_max: 0.391873 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 59 |
+
2026-09-03 11:27:39 - root - INFO - Epoch 23 [train] - train_loss: 0.025192 | val_loss: 0.875133 | lr: 0.000500 | epoch_seconds: 102.041929 | train_seconds: 94.079000 | val_seconds: 7.963000 | steps: 96.000000 | total_steps: 2304.000000 | observations: 69320448.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.091795 | grad_norm_max: 0.508964 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 60 |
+
2026-09-03 11:29:21 - root - INFO - Epoch 24 [train] - train_loss: 0.025227 | val_loss: 0.882931 | lr: 0.000500 | epoch_seconds: 102.111454 | train_seconds: 94.124000 | val_seconds: 7.988000 | steps: 96.000000 | total_steps: 2400.000000 | observations: 72208800.000000 | best_val_loss: 0.792459 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.108911 | grad_norm_max: 0.330561 | clip_fraction: 0.000000 | gpu_peak_gb: 63.209000
|
| 61 |
+
2026-09-03 11:29:21 - root - INFO - Early stopping triggered at epoch 24
|
| 62 |
+
2026-09-03 11:29:21 - root - INFO - Training completed in 2550.31 seconds
|
| 63 |
+
2026-09-03 11:29:21 - root - INFO - Realized optimizer steps: 2400 (96/epoch x 25 epochs) at effective batch 30087 observations
|
| 64 |
+
2026-09-03 11:29:22 - root - INFO - Cost [measured]: 22690.14 TFLOP total (fwd+bwd); 13614.08 TFLOP to best (epoch 15)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_asa_4d_embeddings_s619/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_asa_4d_embeddings_s619",
|
| 3 |
+
"fingerprint": "ecab0f610065386ba094073163753ad8a0baa3a5c2277f0fc00afbce35b85632",
|
| 4 |
+
"checkpoint_sha256": "de36f7beb71d068b47773419eb7f6104d385b5ba29ad43c0d1d281762fbbc283"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e7d16b983a6f7188926d1cf08657d8927cb9b7ceea387c76444c0f7bbac31acf
|
| 3 |
+
size 7101679
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 10723814491392,
|
| 7 |
+
"flops_per_observation": 356426845.19533354,
|
| 8 |
+
"epochs_run": 25,
|
| 9 |
+
"epochs_to_best": 15,
|
| 10 |
+
"optimizer_steps_total": 2400,
|
| 11 |
+
"optimizer_steps_to_best": 1440,
|
| 12 |
+
"observations_total": 72208800,
|
| 13 |
+
"observations_to_best": 43325280,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 2.57371547793408e+16,
|
| 16 |
+
"train_flops_to_best": 1.544229286760448e+16,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "fa_3d",
|
| 20 |
+
"seed": 619
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/logs/metrics.json
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.0645568862867852,
|
| 6 |
+
"val_loss": 0.78973421951135,
|
| 7 |
+
"lr": 0.0003,
|
| 8 |
+
"epoch_seconds": 100.792653799057,
|
| 9 |
+
"train_seconds": 93.363,
|
| 10 |
+
"val_seconds": 7.429,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.78973421951135,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 3.3455607891082764,
|
| 17 |
+
"grad_norm_max": 100.82026672363281,
|
| 18 |
+
"clip_fraction": 0.4270833432674408,
|
| 19 |
+
"gpu_peak_gb": 63.213
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.03389738620414088,
|
| 25 |
+
"val_loss": 0.7931365221738815,
|
| 26 |
+
"lr": 0.0003,
|
| 27 |
+
"epoch_seconds": 98.51388549804688,
|
| 28 |
+
"train_seconds": 90.986,
|
| 29 |
+
"val_seconds": 7.528,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.78973421951135,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.4927580952644348,
|
| 36 |
+
"grad_norm_max": 1.5814059972763062,
|
| 37 |
+
"clip_fraction": 0.0833333358168602,
|
| 38 |
+
"gpu_peak_gb": 63.213
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.031647921908491604,
|
| 44 |
+
"val_loss": 0.792402058839798,
|
| 45 |
+
"lr": 0.0003,
|
| 46 |
+
"epoch_seconds": 98.34855461120605,
|
| 47 |
+
"train_seconds": 91.003,
|
| 48 |
+
"val_seconds": 7.346,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.78973421951135,
|
| 53 |
+
"epochs_without_improvement": 1,
|
| 54 |
+
"grad_norm_mean": 0.3428746461868286,
|
| 55 |
+
"grad_norm_max": 1.3907865285873413,
|
| 56 |
+
"clip_fraction": 0.02083333395421505,
|
| 57 |
+
"gpu_peak_gb": 63.213
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.030849659039328497,
|
| 63 |
+
"val_loss": 0.7982658247152964,
|
| 64 |
+
"lr": 0.0003,
|
| 65 |
+
"epoch_seconds": 98.68772721290588,
|
| 66 |
+
"train_seconds": 91.116,
|
| 67 |
+
"val_seconds": 7.572,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.78973421951135,
|
| 72 |
+
"epochs_without_improvement": 2,
|
| 73 |
+
"grad_norm_mean": 0.32490503787994385,
|
| 74 |
+
"grad_norm_max": 1.111871600151062,
|
| 75 |
+
"clip_fraction": 0.02083333395421505,
|
| 76 |
+
"gpu_peak_gb": 63.213
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.030117057205643505,
|
| 82 |
+
"val_loss": 0.800465834637483,
|
| 83 |
+
"lr": 0.0003,
|
| 84 |
+
"epoch_seconds": 98.39424633979797,
|
| 85 |
+
"train_seconds": 91.11,
|
| 86 |
+
"val_seconds": 7.285,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.78973421951135,
|
| 91 |
+
"epochs_without_improvement": 3,
|
| 92 |
+
"grad_norm_mean": 0.29564493894577026,
|
| 93 |
+
"grad_norm_max": 0.8247587084770203,
|
| 94 |
+
"clip_fraction": 0.0,
|
| 95 |
+
"gpu_peak_gb": 63.213
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.0308314486950015,
|
| 101 |
+
"val_loss": 0.8048372343182564,
|
| 102 |
+
"lr": 0.0003,
|
| 103 |
+
"epoch_seconds": 98.5819640159607,
|
| 104 |
+
"train_seconds": 91.153,
|
| 105 |
+
"val_seconds": 7.429,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.78973421951135,
|
| 110 |
+
"epochs_without_improvement": 4,
|
| 111 |
+
"grad_norm_mean": 0.35723838210105896,
|
| 112 |
+
"grad_norm_max": 0.9427928328514099,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 63.213
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.030148442601785064,
|
| 120 |
+
"val_loss": 0.7964703266819319,
|
| 121 |
+
"lr": 0.00015,
|
| 122 |
+
"epoch_seconds": 98.53008532524109,
|
| 123 |
+
"train_seconds": 91.066,
|
| 124 |
+
"val_seconds": 7.464,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.78973421951135,
|
| 129 |
+
"epochs_without_improvement": 5,
|
| 130 |
+
"grad_norm_mean": 0.3058476448059082,
|
| 131 |
+
"grad_norm_max": 1.1616253852844238,
|
| 132 |
+
"clip_fraction": 0.010416666977107525,
|
| 133 |
+
"gpu_peak_gb": 63.213
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.02858562598703429,
|
| 139 |
+
"val_loss": 0.7911653965711594,
|
| 140 |
+
"lr": 0.00015,
|
| 141 |
+
"epoch_seconds": 98.5530047416687,
|
| 142 |
+
"train_seconds": 91.023,
|
| 143 |
+
"val_seconds": 7.53,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.78973421951135,
|
| 148 |
+
"epochs_without_improvement": 6,
|
| 149 |
+
"grad_norm_mean": 0.16961030662059784,
|
| 150 |
+
"grad_norm_max": 0.8015487790107727,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 63.213
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.02849005345099916,
|
| 158 |
+
"val_loss": 0.7884745250145594,
|
| 159 |
+
"lr": 0.00015,
|
| 160 |
+
"epoch_seconds": 98.5718195438385,
|
| 161 |
+
"train_seconds": 91.011,
|
| 162 |
+
"val_seconds": 7.561,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.7884745250145594,
|
| 167 |
+
"epochs_without_improvement": 7,
|
| 168 |
+
"grad_norm_mean": 0.17472338676452637,
|
| 169 |
+
"grad_norm_max": 0.8850972652435303,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 63.213
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.028187961822065215,
|
| 177 |
+
"val_loss": 0.7851099992791811,
|
| 178 |
+
"lr": 0.00015,
|
| 179 |
+
"epoch_seconds": 98.78246521949768,
|
| 180 |
+
"train_seconds": 91.188,
|
| 181 |
+
"val_seconds": 7.594,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.7851099992791811,
|
| 186 |
+
"epochs_without_improvement": 0,
|
| 187 |
+
"grad_norm_mean": 0.14186134934425354,
|
| 188 |
+
"grad_norm_max": 0.7209665775299072,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 63.213
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.028291786496993154,
|
| 196 |
+
"val_loss": 0.7893072043855985,
|
| 197 |
+
"lr": 0.00015,
|
| 198 |
+
"epoch_seconds": 98.64403367042542,
|
| 199 |
+
"train_seconds": 91.086,
|
| 200 |
+
"val_seconds": 7.558,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.7851099992791811,
|
| 205 |
+
"epochs_without_improvement": 0,
|
| 206 |
+
"grad_norm_mean": 0.17432217299938202,
|
| 207 |
+
"grad_norm_max": 0.6309768557548523,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 63.213
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"epoch": 11,
|
| 213 |
+
"phase": "train",
|
| 214 |
+
"train_loss": 0.028365270661500592,
|
| 215 |
+
"val_loss": 0.7830342824260393,
|
| 216 |
+
"lr": 0.00015,
|
| 217 |
+
"epoch_seconds": 98.63556361198425,
|
| 218 |
+
"train_seconds": 90.946,
|
| 219 |
+
"val_seconds": 7.689,
|
| 220 |
+
"steps": 96,
|
| 221 |
+
"total_steps": 1152,
|
| 222 |
+
"observations": 34660224,
|
| 223 |
+
"best_val_loss": 0.7830342824260393,
|
| 224 |
+
"epochs_without_improvement": 1,
|
| 225 |
+
"grad_norm_mean": 0.18834863603115082,
|
| 226 |
+
"grad_norm_max": 0.7332605123519897,
|
| 227 |
+
"clip_fraction": 0.0,
|
| 228 |
+
"gpu_peak_gb": 63.213
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"epoch": 12,
|
| 232 |
+
"phase": "train",
|
| 233 |
+
"train_loss": 0.028129302760741364,
|
| 234 |
+
"val_loss": 0.7831201950709025,
|
| 235 |
+
"lr": 0.00015,
|
| 236 |
+
"epoch_seconds": 98.5100302696228,
|
| 237 |
+
"train_seconds": 90.969,
|
| 238 |
+
"val_seconds": 7.541,
|
| 239 |
+
"steps": 96,
|
| 240 |
+
"total_steps": 1248,
|
| 241 |
+
"observations": 37548576,
|
| 242 |
+
"best_val_loss": 0.7830342824260393,
|
| 243 |
+
"epochs_without_improvement": 0,
|
| 244 |
+
"grad_norm_mean": 0.16329167783260345,
|
| 245 |
+
"grad_norm_max": 0.6715858578681946,
|
| 246 |
+
"clip_fraction": 0.0,
|
| 247 |
+
"gpu_peak_gb": 63.213
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"epoch": 13,
|
| 251 |
+
"phase": "train",
|
| 252 |
+
"train_loss": 0.028256561413096886,
|
| 253 |
+
"val_loss": 0.7821389908591906,
|
| 254 |
+
"lr": 0.00015,
|
| 255 |
+
"epoch_seconds": 98.59135293960571,
|
| 256 |
+
"train_seconds": 90.956,
|
| 257 |
+
"val_seconds": 7.635,
|
| 258 |
+
"steps": 96,
|
| 259 |
+
"total_steps": 1344,
|
| 260 |
+
"observations": 40436928,
|
| 261 |
+
"best_val_loss": 0.7821389908591906,
|
| 262 |
+
"epochs_without_improvement": 1,
|
| 263 |
+
"grad_norm_mean": 0.1955319046974182,
|
| 264 |
+
"grad_norm_max": 0.6313920021057129,
|
| 265 |
+
"clip_fraction": 0.0,
|
| 266 |
+
"gpu_peak_gb": 63.213
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"epoch": 14,
|
| 270 |
+
"phase": "train",
|
| 271 |
+
"train_loss": 0.027794350947563846,
|
| 272 |
+
"val_loss": 0.7810944989323616,
|
| 273 |
+
"lr": 0.00015,
|
| 274 |
+
"epoch_seconds": 98.76078772544861,
|
| 275 |
+
"train_seconds": 91.041,
|
| 276 |
+
"val_seconds": 7.719,
|
| 277 |
+
"steps": 96,
|
| 278 |
+
"total_steps": 1440,
|
| 279 |
+
"observations": 43325280,
|
| 280 |
+
"best_val_loss": 0.7810944989323616,
|
| 281 |
+
"epochs_without_improvement": 0,
|
| 282 |
+
"grad_norm_mean": 0.15565721690654755,
|
| 283 |
+
"grad_norm_max": 0.965523362159729,
|
| 284 |
+
"clip_fraction": 0.0,
|
| 285 |
+
"gpu_peak_gb": 63.213
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"epoch": 15,
|
| 289 |
+
"phase": "train",
|
| 290 |
+
"train_loss": 0.02786942559760064,
|
| 291 |
+
"val_loss": 0.7880662033955256,
|
| 292 |
+
"lr": 0.00015,
|
| 293 |
+
"epoch_seconds": 98.71872854232788,
|
| 294 |
+
"train_seconds": 91.048,
|
| 295 |
+
"val_seconds": 7.67,
|
| 296 |
+
"steps": 96,
|
| 297 |
+
"total_steps": 1536,
|
| 298 |
+
"observations": 46213632,
|
| 299 |
+
"best_val_loss": 0.7810944989323616,
|
| 300 |
+
"epochs_without_improvement": 0,
|
| 301 |
+
"grad_norm_mean": 0.18099908530712128,
|
| 302 |
+
"grad_norm_max": 0.5677997469902039,
|
| 303 |
+
"clip_fraction": 0.0,
|
| 304 |
+
"gpu_peak_gb": 63.213
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"epoch": 16,
|
| 308 |
+
"phase": "train",
|
| 309 |
+
"train_loss": 0.027670294182219852,
|
| 310 |
+
"val_loss": 0.7926044911146164,
|
| 311 |
+
"lr": 0.00015,
|
| 312 |
+
"epoch_seconds": 98.73964166641235,
|
| 313 |
+
"train_seconds": 91.062,
|
| 314 |
+
"val_seconds": 7.678,
|
| 315 |
+
"steps": 96,
|
| 316 |
+
"total_steps": 1632,
|
| 317 |
+
"observations": 49101984,
|
| 318 |
+
"best_val_loss": 0.7810944989323616,
|
| 319 |
+
"epochs_without_improvement": 1,
|
| 320 |
+
"grad_norm_mean": 0.15730586647987366,
|
| 321 |
+
"grad_norm_max": 0.6617895364761353,
|
| 322 |
+
"clip_fraction": 0.0,
|
| 323 |
+
"gpu_peak_gb": 63.213
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"epoch": 17,
|
| 327 |
+
"phase": "train",
|
| 328 |
+
"train_loss": 0.027391392660016816,
|
| 329 |
+
"val_loss": 0.792871097723643,
|
| 330 |
+
"lr": 0.00015,
|
| 331 |
+
"epoch_seconds": 98.80735182762146,
|
| 332 |
+
"train_seconds": 91.186,
|
| 333 |
+
"val_seconds": 7.621,
|
| 334 |
+
"steps": 96,
|
| 335 |
+
"total_steps": 1728,
|
| 336 |
+
"observations": 51990336,
|
| 337 |
+
"best_val_loss": 0.7810944989323616,
|
| 338 |
+
"epochs_without_improvement": 2,
|
| 339 |
+
"grad_norm_mean": 0.13775110244750977,
|
| 340 |
+
"grad_norm_max": 0.8652346134185791,
|
| 341 |
+
"clip_fraction": 0.0,
|
| 342 |
+
"gpu_peak_gb": 63.213
|
| 343 |
+
},
|
| 344 |
+
{
|
| 345 |
+
"epoch": 18,
|
| 346 |
+
"phase": "train",
|
| 347 |
+
"train_loss": 0.027896205118546884,
|
| 348 |
+
"val_loss": 0.7923415551582972,
|
| 349 |
+
"lr": 0.00015,
|
| 350 |
+
"epoch_seconds": 98.86651349067688,
|
| 351 |
+
"train_seconds": 91.012,
|
| 352 |
+
"val_seconds": 7.854,
|
| 353 |
+
"steps": 96,
|
| 354 |
+
"total_steps": 1824,
|
| 355 |
+
"observations": 54878688,
|
| 356 |
+
"best_val_loss": 0.7810944989323616,
|
| 357 |
+
"epochs_without_improvement": 3,
|
| 358 |
+
"grad_norm_mean": 0.2012912482023239,
|
| 359 |
+
"grad_norm_max": 0.7394346594810486,
|
| 360 |
+
"clip_fraction": 0.0,
|
| 361 |
+
"gpu_peak_gb": 63.213
|
| 362 |
+
},
|
| 363 |
+
{
|
| 364 |
+
"epoch": 19,
|
| 365 |
+
"phase": "train",
|
| 366 |
+
"train_loss": 0.027534089148199808,
|
| 367 |
+
"val_loss": 0.7956177815794945,
|
| 368 |
+
"lr": 0.00015,
|
| 369 |
+
"epoch_seconds": 98.71537017822266,
|
| 370 |
+
"train_seconds": 91.126,
|
| 371 |
+
"val_seconds": 7.59,
|
| 372 |
+
"steps": 96,
|
| 373 |
+
"total_steps": 1920,
|
| 374 |
+
"observations": 57767040,
|
| 375 |
+
"best_val_loss": 0.7810944989323616,
|
| 376 |
+
"epochs_without_improvement": 4,
|
| 377 |
+
"grad_norm_mean": 0.16603727638721466,
|
| 378 |
+
"grad_norm_max": 0.5738664269447327,
|
| 379 |
+
"clip_fraction": 0.0,
|
| 380 |
+
"gpu_peak_gb": 63.213
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"epoch": 20,
|
| 384 |
+
"phase": "train",
|
| 385 |
+
"train_loss": 0.027154293648588162,
|
| 386 |
+
"val_loss": 0.7987843155860901,
|
| 387 |
+
"lr": 7.5e-05,
|
| 388 |
+
"epoch_seconds": 98.68082046508789,
|
| 389 |
+
"train_seconds": 91.088,
|
| 390 |
+
"val_seconds": 7.592,
|
| 391 |
+
"steps": 96,
|
| 392 |
+
"total_steps": 2016,
|
| 393 |
+
"observations": 60655392,
|
| 394 |
+
"best_val_loss": 0.7810944989323616,
|
| 395 |
+
"epochs_without_improvement": 5,
|
| 396 |
+
"grad_norm_mean": 0.13918647170066833,
|
| 397 |
+
"grad_norm_max": 0.568640410900116,
|
| 398 |
+
"clip_fraction": 0.0,
|
| 399 |
+
"gpu_peak_gb": 63.213
|
| 400 |
+
},
|
| 401 |
+
{
|
| 402 |
+
"epoch": 21,
|
| 403 |
+
"phase": "train",
|
| 404 |
+
"train_loss": 0.027027196328466136,
|
| 405 |
+
"val_loss": 0.7962066506346067,
|
| 406 |
+
"lr": 7.5e-05,
|
| 407 |
+
"epoch_seconds": 98.63607549667358,
|
| 408 |
+
"train_seconds": 90.983,
|
| 409 |
+
"val_seconds": 7.653,
|
| 410 |
+
"steps": 96,
|
| 411 |
+
"total_steps": 2112,
|
| 412 |
+
"observations": 63543744,
|
| 413 |
+
"best_val_loss": 0.7810944989323616,
|
| 414 |
+
"epochs_without_improvement": 6,
|
| 415 |
+
"grad_norm_mean": 0.1276547610759735,
|
| 416 |
+
"grad_norm_max": 0.7152206897735596,
|
| 417 |
+
"clip_fraction": 0.0,
|
| 418 |
+
"gpu_peak_gb": 63.213
|
| 419 |
+
},
|
| 420 |
+
{
|
| 421 |
+
"epoch": 22,
|
| 422 |
+
"phase": "train",
|
| 423 |
+
"train_loss": 0.026938745790782075,
|
| 424 |
+
"val_loss": 0.7944365218281746,
|
| 425 |
+
"lr": 7.5e-05,
|
| 426 |
+
"epoch_seconds": 98.59696984291077,
|
| 427 |
+
"train_seconds": 90.99,
|
| 428 |
+
"val_seconds": 7.607,
|
| 429 |
+
"steps": 96,
|
| 430 |
+
"total_steps": 2208,
|
| 431 |
+
"observations": 66432096,
|
| 432 |
+
"best_val_loss": 0.7810944989323616,
|
| 433 |
+
"epochs_without_improvement": 7,
|
| 434 |
+
"grad_norm_mean": 0.1255681961774826,
|
| 435 |
+
"grad_norm_max": 0.5347234606742859,
|
| 436 |
+
"clip_fraction": 0.0,
|
| 437 |
+
"gpu_peak_gb": 63.213
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"epoch": 23,
|
| 441 |
+
"phase": "train",
|
| 442 |
+
"train_loss": 0.026873630937188864,
|
| 443 |
+
"val_loss": 0.7950042312343916,
|
| 444 |
+
"lr": 7.5e-05,
|
| 445 |
+
"epoch_seconds": 98.74189138412476,
|
| 446 |
+
"train_seconds": 91.078,
|
| 447 |
+
"val_seconds": 7.664,
|
| 448 |
+
"steps": 96,
|
| 449 |
+
"total_steps": 2304,
|
| 450 |
+
"observations": 69320448,
|
| 451 |
+
"best_val_loss": 0.7810944989323616,
|
| 452 |
+
"epochs_without_improvement": 8,
|
| 453 |
+
"grad_norm_mean": 0.1123434454202652,
|
| 454 |
+
"grad_norm_max": 0.6965137124061584,
|
| 455 |
+
"clip_fraction": 0.0,
|
| 456 |
+
"gpu_peak_gb": 63.213
|
| 457 |
+
},
|
| 458 |
+
{
|
| 459 |
+
"epoch": 24,
|
| 460 |
+
"phase": "train",
|
| 461 |
+
"train_loss": 0.02682711585657671,
|
| 462 |
+
"val_loss": 0.7942069470882416,
|
| 463 |
+
"lr": 7.5e-05,
|
| 464 |
+
"epoch_seconds": 98.79575538635254,
|
| 465 |
+
"train_seconds": 91.019,
|
| 466 |
+
"val_seconds": 7.777,
|
| 467 |
+
"steps": 96,
|
| 468 |
+
"total_steps": 2400,
|
| 469 |
+
"observations": 72208800,
|
| 470 |
+
"best_val_loss": 0.7810944989323616,
|
| 471 |
+
"epochs_without_improvement": 9,
|
| 472 |
+
"grad_norm_mean": 0.1217893585562706,
|
| 473 |
+
"grad_norm_max": 0.5565298795700073,
|
| 474 |
+
"clip_fraction": 0.0,
|
| 475 |
+
"gpu_peak_gb": 63.213
|
| 476 |
+
}
|
| 477 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/logs/transformer_fa_3d.log
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:34:30 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:34:30 - root - INFO - Model parameters: 1,500,486
|
| 3 |
+
2026-09-03 03:34:30 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 03:36:10 - root - INFO - Epoch 0 [train] - train_loss: 0.064557 | val_loss: 0.789734 | lr: 0.000300 | epoch_seconds: 100.027405 | train_seconds: 92.803000 | val_seconds: 7.224000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 0.000000 | grad_norm_mean: 3.345561 | grad_norm_max: 100.820267 | clip_fraction: 0.427083 | gpu_peak_gb: 63.213000
|
| 5 |
+
2026-09-03 03:37:48 - root - INFO - Epoch 1 [train] - train_loss: 0.033897 | val_loss: 0.793137 | lr: 0.000300 | epoch_seconds: 98.061145 | train_seconds: 91.000000 | val_seconds: 7.061000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.492758 | grad_norm_max: 1.581406 | clip_fraction: 0.083333 | gpu_peak_gb: 63.213000
|
| 6 |
+
2026-09-03 03:39:27 - root - INFO - Epoch 2 [train] - train_loss: 0.031648 | val_loss: 0.792402 | lr: 0.000300 | epoch_seconds: 98.327096 | train_seconds: 91.204000 | val_seconds: 7.123000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.342875 | grad_norm_max: 1.390787 | clip_fraction: 0.020833 | gpu_peak_gb: 63.213000
|
| 7 |
+
2026-09-03 03:41:05 - root - INFO - Epoch 3 [train] - train_loss: 0.030850 | val_loss: 0.798266 | lr: 0.000300 | epoch_seconds: 98.238430 | train_seconds: 91.009000 | val_seconds: 7.229000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.324905 | grad_norm_max: 1.111872 | clip_fraction: 0.020833 | gpu_peak_gb: 63.213000
|
| 8 |
+
2026-09-03 03:42:43 - root - INFO - Epoch 4 [train] - train_loss: 0.030117 | val_loss: 0.800466 | lr: 0.000300 | epoch_seconds: 98.441957 | train_seconds: 91.049000 | val_seconds: 7.393000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.295645 | grad_norm_max: 0.824759 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 9 |
+
2026-09-03 03:44:22 - root - INFO - Epoch 5 [train] - train_loss: 0.030831 | val_loss: 0.804837 | lr: 0.000300 | epoch_seconds: 98.450286 | train_seconds: 91.103000 | val_seconds: 7.348000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.357238 | grad_norm_max: 0.942793 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 10 |
+
2026-09-03 03:46:00 - root - INFO - Epoch 6 [train] - train_loss: 0.030148 | val_loss: 0.796470 | lr: 0.000150 | epoch_seconds: 98.275025 | train_seconds: 91.070000 | val_seconds: 7.205000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.305848 | grad_norm_max: 1.161625 | clip_fraction: 0.010417 | gpu_peak_gb: 63.213000
|
| 11 |
+
2026-09-03 03:47:38 - root - INFO - Epoch 7 [train] - train_loss: 0.028586 | val_loss: 0.791165 | lr: 0.000150 | epoch_seconds: 98.168728 | train_seconds: 91.057000 | val_seconds: 7.112000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.169610 | grad_norm_max: 0.801549 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 12 |
+
2026-09-03 03:49:17 - root - INFO - Epoch 8 [train] - train_loss: 0.028490 | val_loss: 0.788475 | lr: 0.000150 | epoch_seconds: 98.252146 | train_seconds: 91.003000 | val_seconds: 7.249000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.788475 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.174723 | grad_norm_max: 0.885097 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 13 |
+
2026-09-03 03:50:55 - root - INFO - Epoch 9 [train] - train_loss: 0.028188 | val_loss: 0.785110 | lr: 0.000150 | epoch_seconds: 98.363855 | train_seconds: 91.041000 | val_seconds: 7.323000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.785110 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.141861 | grad_norm_max: 0.720967 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 14 |
+
2026-09-03 03:52:33 - root - INFO - Epoch 10 [train] - train_loss: 0.028292 | val_loss: 0.789307 | lr: 0.000150 | epoch_seconds: 98.312318 | train_seconds: 91.013000 | val_seconds: 7.299000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.785110 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.174322 | grad_norm_max: 0.630977 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 15 |
+
2026-09-03 03:54:11 - root - INFO - Epoch 11 [train] - train_loss: 0.028365 | val_loss: 0.783034 | lr: 0.000150 | epoch_seconds: 98.052810 | train_seconds: 90.957000 | val_seconds: 7.096000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.783034 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.188349 | grad_norm_max: 0.733261 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 16 |
+
2026-09-03 03:55:50 - root - INFO - Epoch 12 [train] - train_loss: 0.028129 | val_loss: 0.783120 | lr: 0.000150 | epoch_seconds: 98.581481 | train_seconds: 91.071000 | val_seconds: 7.510000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.783034 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.163292 | grad_norm_max: 0.671586 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 17 |
+
2026-09-03 03:57:28 - root - INFO - Epoch 13 [train] - train_loss: 0.028257 | val_loss: 0.782139 | lr: 0.000150 | epoch_seconds: 98.258147 | train_seconds: 91.147000 | val_seconds: 7.111000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.782139 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.195532 | grad_norm_max: 0.631392 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 18 |
+
2026-09-03 03:59:07 - root - INFO - Epoch 14 [train] - train_loss: 0.027794 | val_loss: 0.781094 | lr: 0.000150 | epoch_seconds: 98.344043 | train_seconds: 91.149000 | val_seconds: 7.195000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.155657 | grad_norm_max: 0.965523 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 19 |
+
2026-09-03 04:00:45 - root - INFO - Epoch 15 [train] - train_loss: 0.027869 | val_loss: 0.788066 | lr: 0.000150 | epoch_seconds: 98.855408 | train_seconds: 91.522000 | val_seconds: 7.333000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.180999 | grad_norm_max: 0.567800 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 20 |
+
2026-09-03 04:02:24 - root - INFO - Epoch 16 [train] - train_loss: 0.027670 | val_loss: 0.792604 | lr: 0.000150 | epoch_seconds: 98.287477 | train_seconds: 91.157000 | val_seconds: 7.130000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.157306 | grad_norm_max: 0.661790 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 21 |
+
2026-09-03 04:04:02 - root - INFO - Epoch 17 [train] - train_loss: 0.027391 | val_loss: 0.792871 | lr: 0.000150 | epoch_seconds: 98.232646 | train_seconds: 91.068000 | val_seconds: 7.165000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.137751 | grad_norm_max: 0.865235 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 22 |
+
2026-09-03 04:05:40 - root - INFO - Epoch 18 [train] - train_loss: 0.027896 | val_loss: 0.792342 | lr: 0.000150 | epoch_seconds: 98.539766 | train_seconds: 91.006000 | val_seconds: 7.533000 | steps: 96.000000 | total_steps: 1824.000000 | observations: 54878688.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.201291 | grad_norm_max: 0.739435 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 23 |
+
2026-09-03 04:07:19 - root - INFO - Epoch 19 [train] - train_loss: 0.027534 | val_loss: 0.795618 | lr: 0.000150 | epoch_seconds: 98.620507 | train_seconds: 91.271000 | val_seconds: 7.350000 | steps: 96.000000 | total_steps: 1920.000000 | observations: 57767040.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.166037 | grad_norm_max: 0.573866 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 24 |
+
2026-09-03 04:08:57 - root - INFO - Epoch 20 [train] - train_loss: 0.027154 | val_loss: 0.798784 | lr: 0.000075 | epoch_seconds: 98.382356 | train_seconds: 91.075000 | val_seconds: 7.307000 | steps: 96.000000 | total_steps: 2016.000000 | observations: 60655392.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.139186 | grad_norm_max: 0.568640 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 25 |
+
2026-09-03 04:10:36 - root - INFO - Epoch 21 [train] - train_loss: 0.027027 | val_loss: 0.796207 | lr: 0.000075 | epoch_seconds: 98.593976 | train_seconds: 91.205000 | val_seconds: 7.389000 | steps: 96.000000 | total_steps: 2112.000000 | observations: 63543744.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.127655 | grad_norm_max: 0.715221 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 26 |
+
2026-09-03 04:12:15 - root - INFO - Epoch 22 [train] - train_loss: 0.026939 | val_loss: 0.794437 | lr: 0.000075 | epoch_seconds: 98.471013 | train_seconds: 91.080000 | val_seconds: 7.391000 | steps: 96.000000 | total_steps: 2208.000000 | observations: 66432096.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.125568 | grad_norm_max: 0.534723 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 27 |
+
2026-09-03 04:13:53 - root - INFO - Epoch 23 [train] - train_loss: 0.026874 | val_loss: 0.795004 | lr: 0.000075 | epoch_seconds: 98.670556 | train_seconds: 91.143000 | val_seconds: 7.527000 | steps: 96.000000 | total_steps: 2304.000000 | observations: 69320448.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.112343 | grad_norm_max: 0.696514 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 28 |
+
2026-09-03 04:15:32 - root - INFO - Epoch 24 [train] - train_loss: 0.026827 | val_loss: 0.794207 | lr: 0.000075 | epoch_seconds: 98.860384 | train_seconds: 91.203000 | val_seconds: 7.658000 | steps: 96.000000 | total_steps: 2400.000000 | observations: 72208800.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.121789 | grad_norm_max: 0.556530 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 29 |
+
2026-09-03 04:15:32 - root - INFO - Early stopping triggered at epoch 24
|
| 30 |
+
2026-09-03 04:15:32 - root - INFO - Training completed in 2461.77 seconds
|
| 31 |
+
2026-09-03 04:15:32 - root - INFO - Realized optimizer steps: 2400 (96/epoch x 25 epochs) at effective batch 30087 observations
|
| 32 |
+
2026-09-03 04:15:34 - root - INFO - Cost [measured]: 25737.15 TFLOP total (fwd+bwd); 15442.29 TFLOP to best (epoch 15)
|
| 33 |
+
2026-09-03 10:46:50 - root - INFO - Trainer initialized on device: cuda
|
| 34 |
+
2026-09-03 10:46:50 - root - INFO - Model parameters: 1,500,486
|
| 35 |
+
2026-09-03 10:46:50 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 36 |
+
2026-09-03 10:48:31 - root - INFO - Epoch 0 [train] - train_loss: 0.064557 | val_loss: 0.789734 | lr: 0.000300 | epoch_seconds: 100.792654 | train_seconds: 93.363000 | val_seconds: 7.429000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 0.000000 | grad_norm_mean: 3.345561 | grad_norm_max: 100.820267 | clip_fraction: 0.427083 | gpu_peak_gb: 63.213000
|
| 37 |
+
2026-09-03 10:50:10 - root - INFO - Epoch 1 [train] - train_loss: 0.033897 | val_loss: 0.793137 | lr: 0.000300 | epoch_seconds: 98.513885 | train_seconds: 90.986000 | val_seconds: 7.528000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.492758 | grad_norm_max: 1.581406 | clip_fraction: 0.083333 | gpu_peak_gb: 63.213000
|
| 38 |
+
2026-09-03 10:51:48 - root - INFO - Epoch 2 [train] - train_loss: 0.031648 | val_loss: 0.792402 | lr: 0.000300 | epoch_seconds: 98.348555 | train_seconds: 91.003000 | val_seconds: 7.346000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.342875 | grad_norm_max: 1.390787 | clip_fraction: 0.020833 | gpu_peak_gb: 63.213000
|
| 39 |
+
2026-09-03 10:53:27 - root - INFO - Epoch 3 [train] - train_loss: 0.030850 | val_loss: 0.798266 | lr: 0.000300 | epoch_seconds: 98.687727 | train_seconds: 91.116000 | val_seconds: 7.572000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.324905 | grad_norm_max: 1.111872 | clip_fraction: 0.020833 | gpu_peak_gb: 63.213000
|
| 40 |
+
2026-09-03 10:55:05 - root - INFO - Epoch 4 [train] - train_loss: 0.030117 | val_loss: 0.800466 | lr: 0.000300 | epoch_seconds: 98.394246 | train_seconds: 91.110000 | val_seconds: 7.285000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.295645 | grad_norm_max: 0.824759 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 41 |
+
2026-09-03 10:56:44 - root - INFO - Epoch 5 [train] - train_loss: 0.030831 | val_loss: 0.804837 | lr: 0.000300 | epoch_seconds: 98.581964 | train_seconds: 91.153000 | val_seconds: 7.429000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.357238 | grad_norm_max: 0.942793 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 42 |
+
2026-09-03 10:58:22 - root - INFO - Epoch 6 [train] - train_loss: 0.030148 | val_loss: 0.796470 | lr: 0.000150 | epoch_seconds: 98.530085 | train_seconds: 91.066000 | val_seconds: 7.464000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.305848 | grad_norm_max: 1.161625 | clip_fraction: 0.010417 | gpu_peak_gb: 63.213000
|
| 43 |
+
2026-09-03 11:00:01 - root - INFO - Epoch 7 [train] - train_loss: 0.028586 | val_loss: 0.791165 | lr: 0.000150 | epoch_seconds: 98.553005 | train_seconds: 91.023000 | val_seconds: 7.530000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.789734 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.169610 | grad_norm_max: 0.801549 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 44 |
+
2026-09-03 11:01:39 - root - INFO - Epoch 8 [train] - train_loss: 0.028490 | val_loss: 0.788475 | lr: 0.000150 | epoch_seconds: 98.571820 | train_seconds: 91.011000 | val_seconds: 7.561000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.788475 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.174723 | grad_norm_max: 0.885097 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 45 |
+
2026-09-03 11:03:18 - root - INFO - Epoch 9 [train] - train_loss: 0.028188 | val_loss: 0.785110 | lr: 0.000150 | epoch_seconds: 98.782465 | train_seconds: 91.188000 | val_seconds: 7.594000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.785110 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.141861 | grad_norm_max: 0.720967 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 46 |
+
2026-09-03 11:04:57 - root - INFO - Epoch 10 [train] - train_loss: 0.028292 | val_loss: 0.789307 | lr: 0.000150 | epoch_seconds: 98.644034 | train_seconds: 91.086000 | val_seconds: 7.558000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.785110 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.174322 | grad_norm_max: 0.630977 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 47 |
+
2026-09-03 11:06:35 - root - INFO - Epoch 11 [train] - train_loss: 0.028365 | val_loss: 0.783034 | lr: 0.000150 | epoch_seconds: 98.635564 | train_seconds: 90.946000 | val_seconds: 7.689000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.783034 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.188349 | grad_norm_max: 0.733261 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 48 |
+
2026-09-03 11:08:14 - root - INFO - Epoch 12 [train] - train_loss: 0.028129 | val_loss: 0.783120 | lr: 0.000150 | epoch_seconds: 98.510030 | train_seconds: 90.969000 | val_seconds: 7.541000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.783034 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.163292 | grad_norm_max: 0.671586 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 49 |
+
2026-09-03 11:09:53 - root - INFO - Epoch 13 [train] - train_loss: 0.028257 | val_loss: 0.782139 | lr: 0.000150 | epoch_seconds: 98.591353 | train_seconds: 90.956000 | val_seconds: 7.635000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.782139 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.195532 | grad_norm_max: 0.631392 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 50 |
+
2026-09-03 11:11:31 - root - INFO - Epoch 14 [train] - train_loss: 0.027794 | val_loss: 0.781094 | lr: 0.000150 | epoch_seconds: 98.760788 | train_seconds: 91.041000 | val_seconds: 7.719000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.155657 | grad_norm_max: 0.965523 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 51 |
+
2026-09-03 11:13:10 - root - INFO - Epoch 15 [train] - train_loss: 0.027869 | val_loss: 0.788066 | lr: 0.000150 | epoch_seconds: 98.718729 | train_seconds: 91.048000 | val_seconds: 7.670000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.180999 | grad_norm_max: 0.567800 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 52 |
+
2026-09-03 11:14:49 - root - INFO - Epoch 16 [train] - train_loss: 0.027670 | val_loss: 0.792604 | lr: 0.000150 | epoch_seconds: 98.739642 | train_seconds: 91.062000 | val_seconds: 7.678000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.157306 | grad_norm_max: 0.661790 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 53 |
+
2026-09-03 11:16:28 - root - INFO - Epoch 17 [train] - train_loss: 0.027391 | val_loss: 0.792871 | lr: 0.000150 | epoch_seconds: 98.807352 | train_seconds: 91.186000 | val_seconds: 7.621000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.137751 | grad_norm_max: 0.865235 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 54 |
+
2026-09-03 11:18:06 - root - INFO - Epoch 18 [train] - train_loss: 0.027896 | val_loss: 0.792342 | lr: 0.000150 | epoch_seconds: 98.866513 | train_seconds: 91.012000 | val_seconds: 7.854000 | steps: 96.000000 | total_steps: 1824.000000 | observations: 54878688.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.201291 | grad_norm_max: 0.739435 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 55 |
+
2026-09-03 11:19:45 - root - INFO - Epoch 19 [train] - train_loss: 0.027534 | val_loss: 0.795618 | lr: 0.000150 | epoch_seconds: 98.715370 | train_seconds: 91.126000 | val_seconds: 7.590000 | steps: 96.000000 | total_steps: 1920.000000 | observations: 57767040.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.166037 | grad_norm_max: 0.573866 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 56 |
+
2026-09-03 11:21:24 - root - INFO - Epoch 20 [train] - train_loss: 0.027154 | val_loss: 0.798784 | lr: 0.000075 | epoch_seconds: 98.680820 | train_seconds: 91.088000 | val_seconds: 7.592000 | steps: 96.000000 | total_steps: 2016.000000 | observations: 60655392.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.139186 | grad_norm_max: 0.568640 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 57 |
+
2026-09-03 11:23:03 - root - INFO - Epoch 21 [train] - train_loss: 0.027027 | val_loss: 0.796207 | lr: 0.000075 | epoch_seconds: 98.636075 | train_seconds: 90.983000 | val_seconds: 7.653000 | steps: 96.000000 | total_steps: 2112.000000 | observations: 63543744.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.127655 | grad_norm_max: 0.715221 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 58 |
+
2026-09-03 11:24:41 - root - INFO - Epoch 22 [train] - train_loss: 0.026939 | val_loss: 0.794437 | lr: 0.000075 | epoch_seconds: 98.596970 | train_seconds: 90.990000 | val_seconds: 7.607000 | steps: 96.000000 | total_steps: 2208.000000 | observations: 66432096.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.125568 | grad_norm_max: 0.534723 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 59 |
+
2026-09-03 11:26:20 - root - INFO - Epoch 23 [train] - train_loss: 0.026874 | val_loss: 0.795004 | lr: 0.000075 | epoch_seconds: 98.741891 | train_seconds: 91.078000 | val_seconds: 7.664000 | steps: 96.000000 | total_steps: 2304.000000 | observations: 69320448.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.112343 | grad_norm_max: 0.696514 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 60 |
+
2026-09-03 11:27:59 - root - INFO - Epoch 24 [train] - train_loss: 0.026827 | val_loss: 0.794207 | lr: 0.000075 | epoch_seconds: 98.795755 | train_seconds: 91.019000 | val_seconds: 7.777000 | steps: 96.000000 | total_steps: 2400.000000 | observations: 72208800.000000 | best_val_loss: 0.781094 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.121789 | grad_norm_max: 0.556530 | clip_fraction: 0.000000 | gpu_peak_gb: 63.213000
|
| 61 |
+
2026-09-03 11:27:59 - root - INFO - Early stopping triggered at epoch 24
|
| 62 |
+
2026-09-03 11:27:59 - root - INFO - Training completed in 2468.31 seconds
|
| 63 |
+
2026-09-03 11:27:59 - root - INFO - Realized optimizer steps: 2400 (96/epoch x 25 epochs) at effective batch 30087 observations
|
| 64 |
+
2026-09-03 11:28:00 - root - INFO - Cost [measured]: 25737.15 TFLOP total (fwd+bwd); 15442.29 TFLOP to best (epoch 15)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_3d_embeddings_s619/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_fa_3d_embeddings_s619",
|
| 3 |
+
"fingerprint": "24352b05bab1898ddeb95fdb29b22a6c22df40dbd1c11a3f0c0d52ecfea030ad",
|
| 4 |
+
"checkpoint_sha256": "e7d16b983a6f7188926d1cf08657d8927cb9b7ceea387c76444c0f7bbac31acf"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8052c3c1fba16042258feb421f0f306f7a6fe9633c053b48e1f990434e529630
|
| 3 |
+
size 7704052
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 10836284951808,
|
| 7 |
+
"flops_per_observation": 360165019.83607537,
|
| 8 |
+
"epochs_run": 18,
|
| 9 |
+
"epochs_to_best": 8,
|
| 10 |
+
"optimizer_steps_total": 1728,
|
| 11 |
+
"optimizer_steps_to_best": 768,
|
| 12 |
+
"observations_total": 51990336,
|
| 13 |
+
"observations_to_best": 23106816,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 1.8725100396724224e+16,
|
| 16 |
+
"train_flops_to_best": 8322266842988544.0,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "fa_4d",
|
| 20 |
+
"seed": 619
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/logs/metrics.json
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.07080395605104665,
|
| 6 |
+
"val_loss": 0.8351183806856474,
|
| 7 |
+
"lr": 0.0003,
|
| 8 |
+
"epoch_seconds": 105.4003095626831,
|
| 9 |
+
"train_seconds": 97.81,
|
| 10 |
+
"val_seconds": 7.59,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.8351183806856474,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 2.3324074745178223,
|
| 17 |
+
"grad_norm_max": 49.75119400024414,
|
| 18 |
+
"clip_fraction": 0.5208333730697632,
|
| 19 |
+
"gpu_peak_gb": 64.592
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.03759916120907292,
|
| 25 |
+
"val_loss": 0.8456980536381403,
|
| 26 |
+
"lr": 0.0003,
|
| 27 |
+
"epoch_seconds": 103.1031723022461,
|
| 28 |
+
"train_seconds": 95.409,
|
| 29 |
+
"val_seconds": 7.694,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.8351183806856474,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.7251629829406738,
|
| 36 |
+
"grad_norm_max": 2.7911102771759033,
|
| 37 |
+
"clip_fraction": 0.2083333432674408,
|
| 38 |
+
"gpu_peak_gb": 64.592
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.030548595008440316,
|
| 44 |
+
"val_loss": 0.8291443387667338,
|
| 45 |
+
"lr": 0.0003,
|
| 46 |
+
"epoch_seconds": 103.32016706466675,
|
| 47 |
+
"train_seconds": 95.495,
|
| 48 |
+
"val_seconds": 7.825,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.8291443387667338,
|
| 53 |
+
"epochs_without_improvement": 1,
|
| 54 |
+
"grad_norm_mean": 0.258899986743927,
|
| 55 |
+
"grad_norm_max": 0.9943273067474365,
|
| 56 |
+
"clip_fraction": 0.0,
|
| 57 |
+
"gpu_peak_gb": 64.592
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.030857934383675456,
|
| 63 |
+
"val_loss": 0.8277755305171013,
|
| 64 |
+
"lr": 0.0003,
|
| 65 |
+
"epoch_seconds": 103.30327844619751,
|
| 66 |
+
"train_seconds": 95.524,
|
| 67 |
+
"val_seconds": 7.779,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.8277755305171013,
|
| 72 |
+
"epochs_without_improvement": 0,
|
| 73 |
+
"grad_norm_mean": 0.33703097701072693,
|
| 74 |
+
"grad_norm_max": 1.0964800119400024,
|
| 75 |
+
"clip_fraction": 0.03125,
|
| 76 |
+
"gpu_peak_gb": 64.592
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.029617769934702665,
|
| 82 |
+
"val_loss": 0.8287606587012609,
|
| 83 |
+
"lr": 0.0003,
|
| 84 |
+
"epoch_seconds": 103.32699036598206,
|
| 85 |
+
"train_seconds": 95.596,
|
| 86 |
+
"val_seconds": 7.731,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.8277755305171013,
|
| 91 |
+
"epochs_without_improvement": 0,
|
| 92 |
+
"grad_norm_mean": 0.2662365436553955,
|
| 93 |
+
"grad_norm_max": 0.805245041847229,
|
| 94 |
+
"clip_fraction": 0.0,
|
| 95 |
+
"gpu_peak_gb": 64.592
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.030110783079483856,
|
| 101 |
+
"val_loss": 0.8261254752675692,
|
| 102 |
+
"lr": 0.0003,
|
| 103 |
+
"epoch_seconds": 103.53298783302307,
|
| 104 |
+
"train_seconds": 95.714,
|
| 105 |
+
"val_seconds": 7.819,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.8261254752675692,
|
| 110 |
+
"epochs_without_improvement": 1,
|
| 111 |
+
"grad_norm_mean": 0.3201768398284912,
|
| 112 |
+
"grad_norm_max": 0.810592532157898,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 64.592
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.02985642010268445,
|
| 120 |
+
"val_loss": 0.823908564945062,
|
| 121 |
+
"lr": 0.0003,
|
| 122 |
+
"epoch_seconds": 103.56659412384033,
|
| 123 |
+
"train_seconds": 95.611,
|
| 124 |
+
"val_seconds": 7.956,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.823908564945062,
|
| 129 |
+
"epochs_without_improvement": 0,
|
| 130 |
+
"grad_norm_mean": 0.308476984500885,
|
| 131 |
+
"grad_norm_max": 1.1725703477859497,
|
| 132 |
+
"clip_fraction": 0.010416666977107525,
|
| 133 |
+
"gpu_peak_gb": 64.592
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.028202748217154294,
|
| 139 |
+
"val_loss": 0.8217176248629888,
|
| 140 |
+
"lr": 0.0003,
|
| 141 |
+
"epoch_seconds": 103.49469709396362,
|
| 142 |
+
"train_seconds": 95.606,
|
| 143 |
+
"val_seconds": 7.889,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.8217176248629888,
|
| 148 |
+
"epochs_without_improvement": 0,
|
| 149 |
+
"grad_norm_mean": 0.1765679568052292,
|
| 150 |
+
"grad_norm_max": 0.8712067604064941,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 64.592
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.028822742286138237,
|
| 158 |
+
"val_loss": 0.8290478860338529,
|
| 159 |
+
"lr": 0.0003,
|
| 160 |
+
"epoch_seconds": 103.68487095832825,
|
| 161 |
+
"train_seconds": 95.714,
|
| 162 |
+
"val_seconds": 7.971,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.8217176248629888,
|
| 167 |
+
"epochs_without_improvement": 0,
|
| 168 |
+
"grad_norm_mean": 0.23305802047252655,
|
| 169 |
+
"grad_norm_max": 0.9813591837882996,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 64.592
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.02813397873736297,
|
| 177 |
+
"val_loss": 0.8263667946060499,
|
| 178 |
+
"lr": 0.0003,
|
| 179 |
+
"epoch_seconds": 103.55544352531433,
|
| 180 |
+
"train_seconds": 95.684,
|
| 181 |
+
"val_seconds": 7.871,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.8217176248629888,
|
| 186 |
+
"epochs_without_improvement": 1,
|
| 187 |
+
"grad_norm_mean": 0.19388268887996674,
|
| 188 |
+
"grad_norm_max": 0.7659673690795898,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 64.592
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.027705710420074563,
|
| 196 |
+
"val_loss": 0.8292938570181528,
|
| 197 |
+
"lr": 0.0003,
|
| 198 |
+
"epoch_seconds": 103.55489540100098,
|
| 199 |
+
"train_seconds": 95.699,
|
| 200 |
+
"val_seconds": 7.856,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.8217176248629888,
|
| 205 |
+
"epochs_without_improvement": 2,
|
| 206 |
+
"grad_norm_mean": 0.1643701195716858,
|
| 207 |
+
"grad_norm_max": 0.5547506809234619,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 64.592
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"epoch": 11,
|
| 213 |
+
"phase": "train",
|
| 214 |
+
"train_loss": 0.027864649756035458,
|
| 215 |
+
"val_loss": 0.8260913317402204,
|
| 216 |
+
"lr": 0.0003,
|
| 217 |
+
"epoch_seconds": 103.4209361076355,
|
| 218 |
+
"train_seconds": 95.612,
|
| 219 |
+
"val_seconds": 7.809,
|
| 220 |
+
"steps": 96,
|
| 221 |
+
"total_steps": 1152,
|
| 222 |
+
"observations": 34660224,
|
| 223 |
+
"best_val_loss": 0.8217176248629888,
|
| 224 |
+
"epochs_without_improvement": 3,
|
| 225 |
+
"grad_norm_mean": 0.18530046939849854,
|
| 226 |
+
"grad_norm_max": 0.8273317813873291,
|
| 227 |
+
"clip_fraction": 0.0,
|
| 228 |
+
"gpu_peak_gb": 64.592
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"epoch": 12,
|
| 232 |
+
"phase": "train",
|
| 233 |
+
"train_loss": 0.027622182678896934,
|
| 234 |
+
"val_loss": 0.8263118465741476,
|
| 235 |
+
"lr": 0.0003,
|
| 236 |
+
"epoch_seconds": 103.4356861114502,
|
| 237 |
+
"train_seconds": 95.706,
|
| 238 |
+
"val_seconds": 7.73,
|
| 239 |
+
"steps": 96,
|
| 240 |
+
"total_steps": 1248,
|
| 241 |
+
"observations": 37548576,
|
| 242 |
+
"best_val_loss": 0.8217176248629888,
|
| 243 |
+
"epochs_without_improvement": 4,
|
| 244 |
+
"grad_norm_mean": 0.17767775058746338,
|
| 245 |
+
"grad_norm_max": 0.5610896944999695,
|
| 246 |
+
"clip_fraction": 0.0,
|
| 247 |
+
"gpu_peak_gb": 64.592
|
| 248 |
+
},
|
| 249 |
+
{
|
| 250 |
+
"epoch": 13,
|
| 251 |
+
"phase": "train",
|
| 252 |
+
"train_loss": 0.0318522320012562,
|
| 253 |
+
"val_loss": 0.8490573142965635,
|
| 254 |
+
"lr": 0.00015,
|
| 255 |
+
"epoch_seconds": 103.52229166030884,
|
| 256 |
+
"train_seconds": 95.605,
|
| 257 |
+
"val_seconds": 7.917,
|
| 258 |
+
"steps": 96,
|
| 259 |
+
"total_steps": 1344,
|
| 260 |
+
"observations": 40436928,
|
| 261 |
+
"best_val_loss": 0.8217176248629888,
|
| 262 |
+
"epochs_without_improvement": 5,
|
| 263 |
+
"grad_norm_mean": 0.4105013310909271,
|
| 264 |
+
"grad_norm_max": 2.575410842895508,
|
| 265 |
+
"clip_fraction": 0.0729166716337204,
|
| 266 |
+
"gpu_peak_gb": 64.592
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"epoch": 14,
|
| 270 |
+
"phase": "train",
|
| 271 |
+
"train_loss": 0.02756207201552267,
|
| 272 |
+
"val_loss": 0.8353298231959343,
|
| 273 |
+
"lr": 0.00015,
|
| 274 |
+
"epoch_seconds": 103.53991436958313,
|
| 275 |
+
"train_seconds": 95.65,
|
| 276 |
+
"val_seconds": 7.89,
|
| 277 |
+
"steps": 96,
|
| 278 |
+
"total_steps": 1440,
|
| 279 |
+
"observations": 43325280,
|
| 280 |
+
"best_val_loss": 0.8217176248629888,
|
| 281 |
+
"epochs_without_improvement": 6,
|
| 282 |
+
"grad_norm_mean": 0.17079666256904602,
|
| 283 |
+
"grad_norm_max": 0.8847362995147705,
|
| 284 |
+
"clip_fraction": 0.0,
|
| 285 |
+
"gpu_peak_gb": 64.592
|
| 286 |
+
},
|
| 287 |
+
{
|
| 288 |
+
"epoch": 15,
|
| 289 |
+
"phase": "train",
|
| 290 |
+
"train_loss": 0.02732320938957855,
|
| 291 |
+
"val_loss": 0.8337483232220014,
|
| 292 |
+
"lr": 0.00015,
|
| 293 |
+
"epoch_seconds": 103.42479991912842,
|
| 294 |
+
"train_seconds": 95.625,
|
| 295 |
+
"val_seconds": 7.799,
|
| 296 |
+
"steps": 96,
|
| 297 |
+
"total_steps": 1536,
|
| 298 |
+
"observations": 46213632,
|
| 299 |
+
"best_val_loss": 0.8217176248629888,
|
| 300 |
+
"epochs_without_improvement": 7,
|
| 301 |
+
"grad_norm_mean": 0.16603055596351624,
|
| 302 |
+
"grad_norm_max": 0.6527597904205322,
|
| 303 |
+
"clip_fraction": 0.0,
|
| 304 |
+
"gpu_peak_gb": 64.592
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"epoch": 16,
|
| 308 |
+
"phase": "train",
|
| 309 |
+
"train_loss": 0.02715208278580879,
|
| 310 |
+
"val_loss": 0.8317332689960798,
|
| 311 |
+
"lr": 0.00015,
|
| 312 |
+
"epoch_seconds": 103.55547022819519,
|
| 313 |
+
"train_seconds": 95.666,
|
| 314 |
+
"val_seconds": 7.89,
|
| 315 |
+
"steps": 96,
|
| 316 |
+
"total_steps": 1632,
|
| 317 |
+
"observations": 49101984,
|
| 318 |
+
"best_val_loss": 0.8217176248629888,
|
| 319 |
+
"epochs_without_improvement": 8,
|
| 320 |
+
"grad_norm_mean": 0.1501699686050415,
|
| 321 |
+
"grad_norm_max": 0.5695124864578247,
|
| 322 |
+
"clip_fraction": 0.0,
|
| 323 |
+
"gpu_peak_gb": 64.592
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"epoch": 17,
|
| 327 |
+
"phase": "train",
|
| 328 |
+
"train_loss": 0.026856552673658978,
|
| 329 |
+
"val_loss": 0.8326841667294502,
|
| 330 |
+
"lr": 0.00015,
|
| 331 |
+
"epoch_seconds": 103.4973554611206,
|
| 332 |
+
"train_seconds": 95.623,
|
| 333 |
+
"val_seconds": 7.874,
|
| 334 |
+
"steps": 96,
|
| 335 |
+
"total_steps": 1728,
|
| 336 |
+
"observations": 51990336,
|
| 337 |
+
"best_val_loss": 0.8217176248629888,
|
| 338 |
+
"epochs_without_improvement": 9,
|
| 339 |
+
"grad_norm_mean": 0.12761624157428741,
|
| 340 |
+
"grad_norm_max": 0.6708542108535767,
|
| 341 |
+
"clip_fraction": 0.0,
|
| 342 |
+
"gpu_peak_gb": 64.592
|
| 343 |
+
}
|
| 344 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/logs/transformer_fa_4d.log
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:41:48 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:41:48 - root - INFO - Model parameters: 1,631,362
|
| 3 |
+
2026-09-03 03:41:48 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 03:43:33 - root - INFO - Epoch 0 [train] - train_loss: 0.070804 | val_loss: 0.835118 | lr: 0.000300 | epoch_seconds: 104.664425 | train_seconds: 97.088000 | val_seconds: 7.576000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.835118 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.332407 | grad_norm_max: 49.751194 | clip_fraction: 0.520833 | gpu_peak_gb: 64.592000
|
| 5 |
+
2026-09-03 03:45:16 - root - INFO - Epoch 1 [train] - train_loss: 0.037599 | val_loss: 0.845698 | lr: 0.000300 | epoch_seconds: 102.772100 | train_seconds: 95.249000 | val_seconds: 7.524000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.835118 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.725163 | grad_norm_max: 2.791110 | clip_fraction: 0.208333 | gpu_peak_gb: 64.592000
|
| 6 |
+
2026-09-03 03:46:58 - root - INFO - Epoch 2 [train] - train_loss: 0.030549 | val_loss: 0.829144 | lr: 0.000300 | epoch_seconds: 102.851844 | train_seconds: 95.436000 | val_seconds: 7.416000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.829144 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.258900 | grad_norm_max: 0.994327 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 7 |
+
2026-09-03 03:48:41 - root - INFO - Epoch 3 [train] - train_loss: 0.030858 | val_loss: 0.827776 | lr: 0.000300 | epoch_seconds: 102.989373 | train_seconds: 95.435000 | val_seconds: 7.555000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.827776 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.337031 | grad_norm_max: 1.096480 | clip_fraction: 0.031250 | gpu_peak_gb: 64.592000
|
| 8 |
+
2026-09-03 03:50:24 - root - INFO - Epoch 4 [train] - train_loss: 0.029618 | val_loss: 0.828761 | lr: 0.000300 | epoch_seconds: 102.907118 | train_seconds: 95.326000 | val_seconds: 7.581000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.827776 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.266237 | grad_norm_max: 0.805245 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 9 |
+
2026-09-03 03:52:07 - root - INFO - Epoch 5 [train] - train_loss: 0.030111 | val_loss: 0.826125 | lr: 0.000300 | epoch_seconds: 102.970324 | train_seconds: 95.334000 | val_seconds: 7.636000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.826125 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.320177 | grad_norm_max: 0.810593 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 10 |
+
2026-09-03 03:53:50 - root - INFO - Epoch 6 [train] - train_loss: 0.029856 | val_loss: 0.823909 | lr: 0.000300 | epoch_seconds: 102.983910 | train_seconds: 95.440000 | val_seconds: 7.544000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.823909 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.308477 | grad_norm_max: 1.172570 | clip_fraction: 0.010417 | gpu_peak_gb: 64.592000
|
| 11 |
+
2026-09-03 03:55:33 - root - INFO - Epoch 7 [train] - train_loss: 0.028203 | val_loss: 0.821718 | lr: 0.000300 | epoch_seconds: 102.836536 | train_seconds: 95.409000 | val_seconds: 7.427000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.176568 | grad_norm_max: 0.871207 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 12 |
+
2026-09-03 03:57:16 - root - INFO - Epoch 8 [train] - train_loss: 0.028823 | val_loss: 0.829048 | lr: 0.000300 | epoch_seconds: 102.910563 | train_seconds: 95.506000 | val_seconds: 7.405000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.233058 | grad_norm_max: 0.981359 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 13 |
+
2026-09-03 03:58:59 - root - INFO - Epoch 9 [train] - train_loss: 0.028134 | val_loss: 0.826367 | lr: 0.000300 | epoch_seconds: 103.175162 | train_seconds: 95.621000 | val_seconds: 7.554000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.193883 | grad_norm_max: 0.765967 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 14 |
+
2026-09-03 04:00:43 - root - INFO - Epoch 10 [train] - train_loss: 0.027706 | val_loss: 0.829294 | lr: 0.000300 | epoch_seconds: 103.292882 | train_seconds: 95.900000 | val_seconds: 7.392000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.164370 | grad_norm_max: 0.554751 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 15 |
+
2026-09-03 04:02:25 - root - INFO - Epoch 11 [train] - train_loss: 0.027865 | val_loss: 0.826091 | lr: 0.000300 | epoch_seconds: 102.699068 | train_seconds: 95.273000 | val_seconds: 7.426000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.185300 | grad_norm_max: 0.827332 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 16 |
+
2026-09-03 04:04:08 - root - INFO - Epoch 12 [train] - train_loss: 0.027622 | val_loss: 0.826312 | lr: 0.000300 | epoch_seconds: 103.026158 | train_seconds: 95.441000 | val_seconds: 7.585000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.177678 | grad_norm_max: 0.561090 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 17 |
+
2026-09-03 04:05:51 - root - INFO - Epoch 13 [train] - train_loss: 0.031852 | val_loss: 0.849057 | lr: 0.000150 | epoch_seconds: 102.836838 | train_seconds: 95.360000 | val_seconds: 7.477000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.410501 | grad_norm_max: 2.575411 | clip_fraction: 0.072917 | gpu_peak_gb: 64.592000
|
| 18 |
+
2026-09-03 04:07:34 - root - INFO - Epoch 14 [train] - train_loss: 0.027562 | val_loss: 0.835330 | lr: 0.000150 | epoch_seconds: 103.097751 | train_seconds: 95.524000 | val_seconds: 7.574000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.170797 | grad_norm_max: 0.884736 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 19 |
+
2026-09-03 04:09:17 - root - INFO - Epoch 15 [train] - train_loss: 0.027323 | val_loss: 0.833748 | lr: 0.000150 | epoch_seconds: 102.904052 | train_seconds: 95.459000 | val_seconds: 7.445000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.166031 | grad_norm_max: 0.652760 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 20 |
+
2026-09-03 04:11:00 - root - INFO - Epoch 16 [train] - train_loss: 0.027152 | val_loss: 0.831733 | lr: 0.000150 | epoch_seconds: 102.900242 | train_seconds: 95.455000 | val_seconds: 7.445000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.150170 | grad_norm_max: 0.569512 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 21 |
+
2026-09-03 04:12:43 - root - INFO - Epoch 17 [train] - train_loss: 0.026857 | val_loss: 0.832684 | lr: 0.000150 | epoch_seconds: 102.982445 | train_seconds: 95.352000 | val_seconds: 7.631000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.127616 | grad_norm_max: 0.670854 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 22 |
+
2026-09-03 04:12:43 - root - INFO - Early stopping triggered at epoch 17
|
| 23 |
+
2026-09-03 04:12:43 - root - INFO - Training completed in 1854.91 seconds
|
| 24 |
+
2026-09-03 04:12:43 - root - INFO - Realized optimizer steps: 1728 (96/epoch x 18 epochs) at effective batch 30087 observations
|
| 25 |
+
2026-09-03 04:12:45 - root - INFO - Cost [measured]: 18725.10 TFLOP total (fwd+bwd); 8322.27 TFLOP to best (epoch 8)
|
| 26 |
+
2026-09-03 10:46:50 - root - INFO - Trainer initialized on device: cuda
|
| 27 |
+
2026-09-03 10:46:50 - root - INFO - Model parameters: 1,631,362
|
| 28 |
+
2026-09-03 10:46:50 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 29 |
+
2026-09-03 10:48:36 - root - INFO - Epoch 0 [train] - train_loss: 0.070804 | val_loss: 0.835118 | lr: 0.000300 | epoch_seconds: 105.400310 | train_seconds: 97.810000 | val_seconds: 7.590000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.835118 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.332407 | grad_norm_max: 49.751194 | clip_fraction: 0.520833 | gpu_peak_gb: 64.592000
|
| 30 |
+
2026-09-03 10:50:19 - root - INFO - Epoch 1 [train] - train_loss: 0.037599 | val_loss: 0.845698 | lr: 0.000300 | epoch_seconds: 103.103172 | train_seconds: 95.409000 | val_seconds: 7.694000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.835118 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.725163 | grad_norm_max: 2.791110 | clip_fraction: 0.208333 | gpu_peak_gb: 64.592000
|
| 31 |
+
2026-09-03 10:52:02 - root - INFO - Epoch 2 [train] - train_loss: 0.030549 | val_loss: 0.829144 | lr: 0.000300 | epoch_seconds: 103.320167 | train_seconds: 95.495000 | val_seconds: 7.825000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.829144 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.258900 | grad_norm_max: 0.994327 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 32 |
+
2026-09-03 10:53:45 - root - INFO - Epoch 3 [train] - train_loss: 0.030858 | val_loss: 0.827776 | lr: 0.000300 | epoch_seconds: 103.303278 | train_seconds: 95.524000 | val_seconds: 7.779000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.827776 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.337031 | grad_norm_max: 1.096480 | clip_fraction: 0.031250 | gpu_peak_gb: 64.592000
|
| 33 |
+
2026-09-03 10:55:29 - root - INFO - Epoch 4 [train] - train_loss: 0.029618 | val_loss: 0.828761 | lr: 0.000300 | epoch_seconds: 103.326990 | train_seconds: 95.596000 | val_seconds: 7.731000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.827776 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.266237 | grad_norm_max: 0.805245 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 34 |
+
2026-09-03 10:57:12 - root - INFO - Epoch 5 [train] - train_loss: 0.030111 | val_loss: 0.826125 | lr: 0.000300 | epoch_seconds: 103.532988 | train_seconds: 95.714000 | val_seconds: 7.819000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.826125 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.320177 | grad_norm_max: 0.810593 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 35 |
+
2026-09-03 10:58:56 - root - INFO - Epoch 6 [train] - train_loss: 0.029856 | val_loss: 0.823909 | lr: 0.000300 | epoch_seconds: 103.566594 | train_seconds: 95.611000 | val_seconds: 7.956000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.823909 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.308477 | grad_norm_max: 1.172570 | clip_fraction: 0.010417 | gpu_peak_gb: 64.592000
|
| 36 |
+
2026-09-03 11:00:39 - root - INFO - Epoch 7 [train] - train_loss: 0.028203 | val_loss: 0.821718 | lr: 0.000300 | epoch_seconds: 103.494697 | train_seconds: 95.606000 | val_seconds: 7.889000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.176568 | grad_norm_max: 0.871207 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 37 |
+
2026-09-03 11:02:23 - root - INFO - Epoch 8 [train] - train_loss: 0.028823 | val_loss: 0.829048 | lr: 0.000300 | epoch_seconds: 103.684871 | train_seconds: 95.714000 | val_seconds: 7.971000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.233058 | grad_norm_max: 0.981359 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 38 |
+
2026-09-03 11:04:07 - root - INFO - Epoch 9 [train] - train_loss: 0.028134 | val_loss: 0.826367 | lr: 0.000300 | epoch_seconds: 103.555444 | train_seconds: 95.684000 | val_seconds: 7.871000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.193883 | grad_norm_max: 0.765967 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 39 |
+
2026-09-03 11:05:50 - root - INFO - Epoch 10 [train] - train_loss: 0.027706 | val_loss: 0.829294 | lr: 0.000300 | epoch_seconds: 103.554895 | train_seconds: 95.699000 | val_seconds: 7.856000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.164370 | grad_norm_max: 0.554751 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 40 |
+
2026-09-03 11:07:34 - root - INFO - Epoch 11 [train] - train_loss: 0.027865 | val_loss: 0.826091 | lr: 0.000300 | epoch_seconds: 103.420936 | train_seconds: 95.612000 | val_seconds: 7.809000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.185300 | grad_norm_max: 0.827332 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 41 |
+
2026-09-03 11:09:17 - root - INFO - Epoch 12 [train] - train_loss: 0.027622 | val_loss: 0.826312 | lr: 0.000300 | epoch_seconds: 103.435686 | train_seconds: 95.706000 | val_seconds: 7.730000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.177678 | grad_norm_max: 0.561090 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 42 |
+
2026-09-03 11:11:01 - root - INFO - Epoch 13 [train] - train_loss: 0.031852 | val_loss: 0.849057 | lr: 0.000150 | epoch_seconds: 103.522292 | train_seconds: 95.605000 | val_seconds: 7.917000 | steps: 96.000000 | total_steps: 1344.000000 | observations: 40436928.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.410501 | grad_norm_max: 2.575411 | clip_fraction: 0.072917 | gpu_peak_gb: 64.592000
|
| 43 |
+
2026-09-03 11:12:44 - root - INFO - Epoch 14 [train] - train_loss: 0.027562 | val_loss: 0.835330 | lr: 0.000150 | epoch_seconds: 103.539914 | train_seconds: 95.650000 | val_seconds: 7.890000 | steps: 96.000000 | total_steps: 1440.000000 | observations: 43325280.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.170797 | grad_norm_max: 0.884736 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 44 |
+
2026-09-03 11:14:28 - root - INFO - Epoch 15 [train] - train_loss: 0.027323 | val_loss: 0.833748 | lr: 0.000150 | epoch_seconds: 103.424800 | train_seconds: 95.625000 | val_seconds: 7.799000 | steps: 96.000000 | total_steps: 1536.000000 | observations: 46213632.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.166031 | grad_norm_max: 0.652760 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 45 |
+
2026-09-03 11:16:11 - root - INFO - Epoch 16 [train] - train_loss: 0.027152 | val_loss: 0.831733 | lr: 0.000150 | epoch_seconds: 103.555470 | train_seconds: 95.666000 | val_seconds: 7.890000 | steps: 96.000000 | total_steps: 1632.000000 | observations: 49101984.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.150170 | grad_norm_max: 0.569512 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 46 |
+
2026-09-03 11:17:55 - root - INFO - Epoch 17 [train] - train_loss: 0.026857 | val_loss: 0.832684 | lr: 0.000150 | epoch_seconds: 103.497355 | train_seconds: 95.623000 | val_seconds: 7.874000 | steps: 96.000000 | total_steps: 1728.000000 | observations: 51990336.000000 | best_val_loss: 0.821718 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.127616 | grad_norm_max: 0.670854 | clip_fraction: 0.000000 | gpu_peak_gb: 64.592000
|
| 47 |
+
2026-09-03 11:17:55 - root - INFO - Early stopping triggered at epoch 17
|
| 48 |
+
2026-09-03 11:17:55 - root - INFO - Training completed in 1864.35 seconds
|
| 49 |
+
2026-09-03 11:17:55 - root - INFO - Realized optimizer steps: 1728 (96/epoch x 18 epochs) at effective batch 30087 observations
|
| 50 |
+
2026-09-03 11:17:57 - root - INFO - Cost [measured]: 18725.10 TFLOP total (fwd+bwd); 8322.27 TFLOP to best (epoch 8)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill/transformer_fa_4d_embeddings_s619/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_fa_4d_embeddings_s619",
|
| 3 |
+
"fingerprint": "3782901dd498209f2d625a792cb428f1682f0a14549fb250ce51298cd5a482de",
|
| 4 |
+
"checkpoint_sha256": "8052c3c1fba16042258feb421f0f306f7a6fe9633c053b48e1f990434e529630"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s619.log
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA extension for structured kernels (Cauchy and Vandermonde multiplication) not found. Install by going to extensions/kernels/ and running `python setup.py install`, for improved speed and memory efficiency. Note that the kernel changed for state-spaces 4.0 and must be recompiled.
|
| 2 |
+
Falling back on slow Cauchy and Vandermonde kernel. Install at least one of pykeops or the CUDA extension for better speed and memory efficiency.
|
| 3 |
+
/root/autodl-tmp/mlsf/src/models/transformer.py:242: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
|
| 4 |
+
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)
|
| 5 |
+
================================================================================
|
| 6 |
+
Training transformer with asa_2d variant (seed=619)
|
| 7 |
+
================================================================================
|
| 8 |
+
Loading configurations...
|
| 9 |
+
Loaded model config from config/models/transformer.yaml
|
| 10 |
+
Loaded variant config from config/variants/asa_2d.yaml
|
| 11 |
+
Loaded data config from config/census.yaml
|
| 12 |
+
Configuration loaded
|
| 13 |
+
Model: transformer
|
| 14 |
+
Variant: asa_2d
|
| 15 |
+
Batch size: 1
|
| 16 |
+
Epochs: 200
|
| 17 |
+
Learning rate: 0.001
|
| 18 |
+
|
| 19 |
+
Loading and preprocessing data...
|
| 20 |
+
[census] 30,087 series | grid 1343x14x2 | 192 months | 9 channels | F=35 (lag_mode=agg) | combo/sample ~ 152 MB
|
| 21 |
+
|
| 22 |
+
Creating dataloaders (combo=True, num_workers=4 from $CENSUS_COMBO_NUM_WORKERS)...
|
| 23 |
+
[census] dataloaders (combo): 96 train, 24 val, 24 test batches
|
| 24 |
+
Creating model: transformer...
|
| 25 |
+
Model created with 1,003,374 parameters
|
| 26 |
+
|
| 27 |
+
Initializing trainer...
|
| 28 |
+
Starting training...
|
| 29 |
+
--------------------------------------------------------------------------------
|
| 30 |
+
2026-09-03 03:58:09 - root - INFO - Trainer initialized on device: cuda
|
| 31 |
+
2026-09-03 03:58:09 - root - INFO - Model parameters: 1,003,374
|
| 32 |
+
2026-09-03 03:58:09 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 33 |
+
/root/miniconda3/lib/python3.12/site-packages/torch/autograd/graph.py:882: UserWarning: Memory Efficient attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:900.)
|
| 34 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 35 |
+
2026-09-03 03:59:25 - root - INFO - Epoch 0 [train] - train_loss: 0.087173 | val_loss: 0.788637 | lr: 0.001000 | epoch_seconds: 75.990085 | train_seconds: 70.732000 | val_seconds: 5.258000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 0.000000 | grad_norm_mean: 1.955022 | grad_norm_max: 30.717197 | clip_fraction: 0.562500 | gpu_peak_gb: 49.363000
|
| 36 |
+
✓ Saved best model to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/best.pth
|
| 37 |
+
2026-09-03 04:00:40 - root - INFO - Epoch 1 [train] - train_loss: 0.037517 | val_loss: 0.804539 | lr: 0.001000 | epoch_seconds: 74.849554 | train_seconds: 69.383000 | val_seconds: 5.466000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.615605 | grad_norm_max: 2.214387 | clip_fraction: 0.104167 | gpu_peak_gb: 49.363000
|
| 38 |
+
2026-09-03 04:01:55 - root - INFO - Epoch 2 [train] - train_loss: 0.032341 | val_loss: 0.805234 | lr: 0.001000 | epoch_seconds: 74.449460 | train_seconds: 69.034000 | val_seconds: 5.415000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.355096 | grad_norm_max: 1.685804 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 39 |
+
2026-09-03 04:03:09 - root - INFO - Epoch 3 [train] - train_loss: 0.031777 | val_loss: 0.828920 | lr: 0.001000 | epoch_seconds: 74.181875 | train_seconds: 68.860000 | val_seconds: 5.321000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.406957 | grad_norm_max: 1.400108 | clip_fraction: 0.041667 | gpu_peak_gb: 49.363000
|
| 40 |
+
2026-09-03 04:04:23 - root - INFO - Epoch 4 [train] - train_loss: 0.029820 | val_loss: 0.836971 | lr: 0.001000 | epoch_seconds: 74.239392 | train_seconds: 68.978000 | val_seconds: 5.261000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.288255 | grad_norm_max: 0.876655 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 41 |
+
2026-09-03 04:05:37 - root - INFO - Epoch 5 [train] - train_loss: 0.030535 | val_loss: 0.850426 | lr: 0.001000 | epoch_seconds: 74.254848 | train_seconds: 68.950000 | val_seconds: 5.305000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.362465 | grad_norm_max: 0.911757 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 42 |
+
2026-09-03 04:06:52 - root - INFO - Epoch 6 [train] - train_loss: 0.029409 | val_loss: 0.865797 | lr: 0.000500 | epoch_seconds: 74.338067 | train_seconds: 68.899000 | val_seconds: 5.439000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.287198 | grad_norm_max: 1.145041 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 43 |
+
2026-09-03 04:08:06 - root - INFO - Epoch 7 [train] - train_loss: 0.027890 | val_loss: 0.872928 | lr: 0.000500 | epoch_seconds: 74.226032 | train_seconds: 68.865000 | val_seconds: 5.361000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.167599 | grad_norm_max: 0.740990 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 44 |
+
2026-09-03 04:09:20 - root - INFO - Epoch 8 [train] - train_loss: 0.027714 | val_loss: 0.888542 | lr: 0.000500 | epoch_seconds: 74.183996 | train_seconds: 68.875000 | val_seconds: 5.309000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.180803 | grad_norm_max: 0.881730 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 45 |
+
2026-09-03 04:10:34 - root - INFO - Epoch 9 [train] - train_loss: 0.027235 | val_loss: 0.891802 | lr: 0.000500 | epoch_seconds: 74.194466 | train_seconds: 68.870000 | val_seconds: 5.324000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.147411 | grad_norm_max: 0.598976 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 46 |
+
2026-09-03 04:11:49 - root - INFO - Epoch 10 [train] - train_loss: 0.027124 | val_loss: 0.895738 | lr: 0.000500 | epoch_seconds: 74.357629 | train_seconds: 68.902000 | val_seconds: 5.456000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.154228 | grad_norm_max: 0.606047 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 47 |
+
2026-09-03 04:11:49 - root - INFO - Early stopping triggered at epoch 10
|
| 48 |
+
2026-09-03 04:11:49 - root - INFO - Training completed in 819.28 seconds
|
| 49 |
+
2026-09-03 04:11:49 - root - INFO - Realized optimizer steps: 1056 (96/epoch x 11 epochs) at effective batch 30087 observations
|
| 50 |
+
2026-09-03 04:11:50 - root - INFO - Cost [measured]: 6365.74 TFLOP total (fwd+bwd); 578.70 TFLOP to best (epoch 1)
|
| 51 |
+
--------------------------------------------------------------------------------
|
| 52 |
+
Training completed!
|
| 53 |
+
|
| 54 |
+
Final train loss: 0.027124
|
| 55 |
+
Final val loss: 0.895738
|
| 56 |
+
|
| 57 |
+
Best model saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/best.pth
|
| 58 |
+
Logs saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/logs
|
| 59 |
+
Cost row saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/cost.json
|
| 60 |
+
Loss curve saved to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/plots/loss_curve.png
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s732.log
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA extension for structured kernels (Cauchy and Vandermonde multiplication) not found. Install by going to extensions/kernels/ and running `python setup.py install`, for improved speed and memory efficiency. Note that the kernel changed for state-spaces 4.0 and must be recompiled.
|
| 2 |
+
Falling back on slow Cauchy and Vandermonde kernel. Install at least one of pykeops or the CUDA extension for better speed and memory efficiency.
|
| 3 |
+
/root/autodl-tmp/mlsf/src/models/transformer.py:242: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
|
| 4 |
+
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)
|
| 5 |
+
================================================================================
|
| 6 |
+
Training transformer with asa_2d variant (seed=732)
|
| 7 |
+
================================================================================
|
| 8 |
+
Loading configurations...
|
| 9 |
+
Loaded model config from config/models/transformer.yaml
|
| 10 |
+
Loaded variant config from config/variants/asa_2d.yaml
|
| 11 |
+
Loaded data config from config/census.yaml
|
| 12 |
+
Configuration loaded
|
| 13 |
+
Model: transformer
|
| 14 |
+
Variant: asa_2d
|
| 15 |
+
Batch size: 1
|
| 16 |
+
Epochs: 200
|
| 17 |
+
Learning rate: 0.001
|
| 18 |
+
|
| 19 |
+
Loading and preprocessing data...
|
| 20 |
+
[census] 30,087 series | grid 1343x14x2 | 192 months | 9 channels | F=35 (lag_mode=agg) | combo/sample ~ 152 MB
|
| 21 |
+
|
| 22 |
+
Creating dataloaders (combo=True, num_workers=4 from $CENSUS_COMBO_NUM_WORKERS)...
|
| 23 |
+
[census] dataloaders (combo): 96 train, 24 val, 24 test batches
|
| 24 |
+
Creating model: transformer...
|
| 25 |
+
Model created with 1,003,374 parameters
|
| 26 |
+
|
| 27 |
+
Initializing trainer...
|
| 28 |
+
Starting training...
|
| 29 |
+
--------------------------------------------------------------------------------
|
| 30 |
+
2026-09-03 11:29:37 - root - INFO - Trainer initialized on device: cuda
|
| 31 |
+
2026-09-03 11:29:37 - root - INFO - Model parameters: 1,003,374
|
| 32 |
+
2026-09-03 11:29:37 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 33 |
+
/root/miniconda3/lib/python3.12/site-packages/torch/autograd/graph.py:882: UserWarning: Memory Efficient attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:900.)
|
| 34 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 35 |
+
2026-09-03 11:30:53 - root - INFO - Epoch 0 [train] - train_loss: 0.090866 | val_loss: 0.890958 | lr: 0.001000 | epoch_seconds: 76.470029 | train_seconds: 70.916000 | val_seconds: 5.554000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.890958 | epochs_without_improvement: 0.000000 | grad_norm_mean: 1.906204 | grad_norm_max: 18.913582 | clip_fraction: 0.468750 | gpu_peak_gb: 49.363000
|
| 36 |
+
✓ Saved best model to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth
|
| 37 |
+
2026-09-03 11:32:08 - root - INFO - Epoch 1 [train] - train_loss: 0.036332 | val_loss: 0.877109 | lr: 0.001000 | epoch_seconds: 74.575799 | train_seconds: 69.044000 | val_seconds: 5.532000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.877109 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.474631 | grad_norm_max: 1.359816 | clip_fraction: 0.052083 | gpu_peak_gb: 49.363000
|
| 38 |
+
✓ Saved best model to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth
|
| 39 |
+
2026-09-03 11:33:22 - root - INFO - Epoch 2 [train] - train_loss: 0.032410 | val_loss: 0.876547 | lr: 0.001000 | epoch_seconds: 74.473110 | train_seconds: 69.015000 | val_seconds: 5.458000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.345095 | grad_norm_max: 1.051386 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 40 |
+
✓ Saved best model to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth
|
| 41 |
+
2026-09-03 11:34:37 - root - INFO - Epoch 3 [train] - train_loss: 0.031019 | val_loss: 0.881686 | lr: 0.001000 | epoch_seconds: 74.485544 | train_seconds: 69.056000 | val_seconds: 5.429000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.278520 | grad_norm_max: 0.795859 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 42 |
+
2026-09-03 11:35:51 - root - INFO - Epoch 4 [train] - train_loss: 0.030296 | val_loss: 0.888734 | lr: 0.001000 | epoch_seconds: 74.650599 | train_seconds: 69.133000 | val_seconds: 5.518000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.251237 | grad_norm_max: 1.107619 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 43 |
+
2026-09-03 11:37:06 - root - INFO - Epoch 5 [train] - train_loss: 0.028707 | val_loss: 0.888584 | lr: 0.001000 | epoch_seconds: 74.434504 | train_seconds: 69.030000 | val_seconds: 5.404000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.185404 | grad_norm_max: 0.849419 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 44 |
+
2026-09-03 11:38:20 - root - INFO - Epoch 6 [train] - train_loss: 0.028031 | val_loss: 0.889256 | lr: 0.001000 | epoch_seconds: 74.645522 | train_seconds: 68.996000 | val_seconds: 5.650000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.179586 | grad_norm_max: 0.505556 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 45 |
+
2026-09-03 11:39:35 - root - INFO - Epoch 7 [train] - train_loss: 0.028063 | val_loss: 0.891378 | lr: 0.001000 | epoch_seconds: 74.706357 | train_seconds: 69.189000 | val_seconds: 5.517000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.217270 | grad_norm_max: 0.675582 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 46 |
+
2026-09-03 11:40:50 - root - INFO - Epoch 8 [train] - train_loss: 0.027260 | val_loss: 0.889347 | lr: 0.000500 | epoch_seconds: 74.442591 | train_seconds: 68.997000 | val_seconds: 5.445000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.169286 | grad_norm_max: 0.502799 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 47 |
+
2026-09-03 11:42:04 - root - INFO - Epoch 9 [train] - train_loss: 0.026545 | val_loss: 0.887272 | lr: 0.000500 | epoch_seconds: 74.551551 | train_seconds: 69.051000 | val_seconds: 5.500000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.138316 | grad_norm_max: 0.412564 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 48 |
+
2026-09-03 11:43:18 - root - INFO - Epoch 10 [train] - train_loss: 0.026303 | val_loss: 0.883542 | lr: 0.000500 | epoch_seconds: 74.402445 | train_seconds: 68.964000 | val_seconds: 5.438000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.126705 | grad_norm_max: 0.530019 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 49 |
+
2026-09-03 11:44:33 - root - INFO - Epoch 11 [train] - train_loss: 0.025964 | val_loss: 0.881768 | lr: 0.000500 | epoch_seconds: 74.563666 | train_seconds: 69.077000 | val_seconds: 5.487000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.104789 | grad_norm_max: 0.395686 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 50 |
+
2026-09-03 11:45:48 - root - INFO - Epoch 12 [train] - train_loss: 0.026037 | val_loss: 0.878387 | lr: 0.000500 | epoch_seconds: 74.628792 | train_seconds: 69.075000 | val_seconds: 5.554000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.122913 | grad_norm_max: 0.488789 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 51 |
+
2026-09-03 11:45:48 - root - INFO - Early stopping triggered at epoch 12
|
| 52 |
+
2026-09-03 11:45:48 - root - INFO - Training completed in 971.07 seconds
|
| 53 |
+
2026-09-03 11:45:48 - root - INFO - Realized optimizer steps: 1248 (96/epoch x 13 epochs) at effective batch 30087 observations
|
| 54 |
+
2026-09-03 11:45:49 - root - INFO - Cost [measured]: 7523.15 TFLOP total (fwd+bwd); 1736.11 TFLOP to best (epoch 3)
|
| 55 |
+
--------------------------------------------------------------------------------
|
| 56 |
+
Training completed!
|
| 57 |
+
|
| 58 |
+
Final train loss: 0.026037
|
| 59 |
+
Final val loss: 0.878387
|
| 60 |
+
|
| 61 |
+
Best model saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth
|
| 62 |
+
Logs saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/logs
|
| 63 |
+
Cost row saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/cost.json
|
| 64 |
+
Loss curve saved to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/plots/loss_curve.png
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/logs/transformer_asa_2d_id_embeddings_s947.log
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA extension for structured kernels (Cauchy and Vandermonde multiplication) not found. Install by going to extensions/kernels/ and running `python setup.py install`, for improved speed and memory efficiency. Note that the kernel changed for state-spaces 4.0 and must be recompiled.
|
| 2 |
+
Falling back on slow Cauchy and Vandermonde kernel. Install at least one of pykeops or the CUDA extension for better speed and memory efficiency.
|
| 3 |
+
/root/autodl-tmp/mlsf/src/models/transformer.py:242: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
|
| 4 |
+
self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers)
|
| 5 |
+
================================================================================
|
| 6 |
+
Training transformer with asa_2d variant (seed=947)
|
| 7 |
+
================================================================================
|
| 8 |
+
Loading configurations...
|
| 9 |
+
Loaded model config from config/models/transformer.yaml
|
| 10 |
+
Loaded variant config from config/variants/asa_2d.yaml
|
| 11 |
+
Loaded data config from config/census.yaml
|
| 12 |
+
Configuration loaded
|
| 13 |
+
Model: transformer
|
| 14 |
+
Variant: asa_2d
|
| 15 |
+
Batch size: 1
|
| 16 |
+
Epochs: 200
|
| 17 |
+
Learning rate: 0.001
|
| 18 |
+
|
| 19 |
+
Loading and preprocessing data...
|
| 20 |
+
[census] 30,087 series | grid 1343x14x2 | 192 months | 9 channels | F=35 (lag_mode=agg) | combo/sample ~ 152 MB
|
| 21 |
+
|
| 22 |
+
Creating dataloaders (combo=True, num_workers=4 from $CENSUS_COMBO_NUM_WORKERS)...
|
| 23 |
+
[census] dataloaders (combo): 96 train, 24 val, 24 test batches
|
| 24 |
+
Creating model: transformer...
|
| 25 |
+
Model created with 1,003,374 parameters
|
| 26 |
+
|
| 27 |
+
Initializing trainer...
|
| 28 |
+
Starting training...
|
| 29 |
+
--------------------------------------------------------------------------------
|
| 30 |
+
2026-09-03 11:29:37 - root - INFO - Trainer initialized on device: cuda
|
| 31 |
+
2026-09-03 11:29:37 - root - INFO - Model parameters: 1,003,374
|
| 32 |
+
2026-09-03 11:29:37 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 33 |
+
/root/miniconda3/lib/python3.12/site-packages/torch/autograd/graph.py:882: UserWarning: Memory Efficient attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:900.)
|
| 34 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 35 |
+
2026-09-03 11:30:54 - root - INFO - Epoch 0 [train] - train_loss: 0.094464 | val_loss: 0.805272 | lr: 0.001000 | epoch_seconds: 77.065206 | train_seconds: 71.620000 | val_seconds: 5.445000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.047662 | grad_norm_max: 42.005630 | clip_fraction: 0.458333 | gpu_peak_gb: 49.363000
|
| 36 |
+
✓ Saved best model to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/best.pth
|
| 37 |
+
2026-09-03 11:32:09 - root - INFO - Epoch 1 [train] - train_loss: 0.035635 | val_loss: 0.822844 | lr: 0.001000 | epoch_seconds: 75.260048 | train_seconds: 69.725000 | val_seconds: 5.535000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.505975 | grad_norm_max: 1.507473 | clip_fraction: 0.104167 | gpu_peak_gb: 49.363000
|
| 38 |
+
2026-09-03 11:33:24 - root - INFO - Epoch 2 [train] - train_loss: 0.033278 | val_loss: 0.825155 | lr: 0.001000 | epoch_seconds: 75.282693 | train_seconds: 69.580000 | val_seconds: 5.702000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.463592 | grad_norm_max: 1.544577 | clip_fraction: 0.041667 | gpu_peak_gb: 49.363000
|
| 39 |
+
2026-09-03 11:34:39 - root - INFO - Epoch 3 [train] - train_loss: 0.031410 | val_loss: 0.822576 | lr: 0.001000 | epoch_seconds: 75.026498 | train_seconds: 69.546000 | val_seconds: 5.481000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.347183 | grad_norm_max: 1.201439 | clip_fraction: 0.020833 | gpu_peak_gb: 49.363000
|
| 40 |
+
2026-09-03 11:35:55 - root - INFO - Epoch 4 [train] - train_loss: 0.030225 | val_loss: 0.822740 | lr: 0.001000 | epoch_seconds: 75.150697 | train_seconds: 69.635000 | val_seconds: 5.515000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.299757 | grad_norm_max: 1.329149 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 41 |
+
2026-09-03 11:37:10 - root - INFO - Epoch 5 [train] - train_loss: 0.029317 | val_loss: 0.835133 | lr: 0.001000 | epoch_seconds: 75.034961 | train_seconds: 69.566000 | val_seconds: 5.469000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.260603 | grad_norm_max: 0.772015 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 42 |
+
2026-09-03 11:38:25 - root - INFO - Epoch 6 [train] - train_loss: 0.029724 | val_loss: 0.865611 | lr: 0.000500 | epoch_seconds: 75.246378 | train_seconds: 69.741000 | val_seconds: 5.506000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.315095 | grad_norm_max: 1.238032 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 43 |
+
2026-09-03 11:39:40 - root - INFO - Epoch 7 [train] - train_loss: 0.028099 | val_loss: 0.859405 | lr: 0.000500 | epoch_seconds: 75.145617 | train_seconds: 69.658000 | val_seconds: 5.488000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.180556 | grad_norm_max: 0.838833 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 44 |
+
2026-09-03 11:40:55 - root - INFO - Epoch 8 [train] - train_loss: 0.027842 | val_loss: 0.860088 | lr: 0.000500 | epoch_seconds: 75.105267 | train_seconds: 69.519000 | val_seconds: 5.587000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.188690 | grad_norm_max: 0.740513 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 45 |
+
2026-09-03 11:42:10 - root - INFO - Epoch 9 [train] - train_loss: 0.027632 | val_loss: 0.859373 | lr: 0.000500 | epoch_seconds: 75.162982 | train_seconds: 69.574000 | val_seconds: 5.589000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.191893 | grad_norm_max: 0.497907 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 46 |
+
2026-09-03 11:43:25 - root - INFO - Epoch 10 [train] - train_loss: 0.027107 | val_loss: 0.856120 | lr: 0.000500 | epoch_seconds: 75.142699 | train_seconds: 69.575000 | val_seconds: 5.568000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.138060 | grad_norm_max: 0.561515 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 47 |
+
2026-09-03 11:43:25 - root - INFO - Early stopping triggered at epoch 10
|
| 48 |
+
2026-09-03 11:43:25 - root - INFO - Training completed in 828.64 seconds
|
| 49 |
+
2026-09-03 11:43:25 - root - INFO - Realized optimizer steps: 1056 (96/epoch x 11 epochs) at effective batch 30087 observations
|
| 50 |
+
2026-09-03 11:43:27 - root - INFO - Cost [measured]: 6365.74 TFLOP total (fwd+bwd); 578.70 TFLOP to best (epoch 1)
|
| 51 |
+
--------------------------------------------------------------------------------
|
| 52 |
+
Training completed!
|
| 53 |
+
|
| 54 |
+
Final train loss: 0.027107
|
| 55 |
+
Final val loss: 0.856120
|
| 56 |
+
|
| 57 |
+
Best model saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/best.pth
|
| 58 |
+
Logs saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/logs
|
| 59 |
+
Cost row saved to: /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/cost.json
|
| 60 |
+
Loss curve saved to /root/autodl-tmp/mlsf/outputs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/plots/loss_curve.png
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/manifest.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:74efe6fa3803d496fd8fd67e38f213f16bf843a9b7abde976728a1b182812c90
|
| 3 |
+
size 5089325
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 6028167585024,
|
| 7 |
+
"flops_per_observation": 200357881.6440323,
|
| 8 |
+
"epochs_run": 11,
|
| 9 |
+
"epochs_to_best": 1,
|
| 10 |
+
"optimizer_steps_total": 1056,
|
| 11 |
+
"optimizer_steps_to_best": 96,
|
| 12 |
+
"observations_total": 31771872,
|
| 13 |
+
"observations_to_best": 2888352,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 6365744969785344.0,
|
| 16 |
+
"train_flops_to_best": 578704088162304.0,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "asa_2d",
|
| 20 |
+
"seed": 619
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/logs/metrics.json
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.08717310176386188,
|
| 6 |
+
"val_loss": 0.7886371314525604,
|
| 7 |
+
"lr": 0.001,
|
| 8 |
+
"epoch_seconds": 75.99008464813232,
|
| 9 |
+
"train_seconds": 70.732,
|
| 10 |
+
"val_seconds": 5.258,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.7886371314525604,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 1.9550222158432007,
|
| 17 |
+
"grad_norm_max": 30.71719741821289,
|
| 18 |
+
"clip_fraction": 0.5625,
|
| 19 |
+
"gpu_peak_gb": 49.363
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.03751727562242498,
|
| 25 |
+
"val_loss": 0.8045391316215197,
|
| 26 |
+
"lr": 0.001,
|
| 27 |
+
"epoch_seconds": 74.84955430030823,
|
| 28 |
+
"train_seconds": 69.383,
|
| 29 |
+
"val_seconds": 5.466,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.7886371314525604,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.6156046986579895,
|
| 36 |
+
"grad_norm_max": 2.2143871784210205,
|
| 37 |
+
"clip_fraction": 0.1041666716337204,
|
| 38 |
+
"gpu_peak_gb": 49.363
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.032341348240152,
|
| 44 |
+
"val_loss": 0.8052342484394709,
|
| 45 |
+
"lr": 0.001,
|
| 46 |
+
"epoch_seconds": 74.44945955276489,
|
| 47 |
+
"train_seconds": 69.034,
|
| 48 |
+
"val_seconds": 5.415,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.7886371314525604,
|
| 53 |
+
"epochs_without_improvement": 1,
|
| 54 |
+
"grad_norm_mean": 0.3550960421562195,
|
| 55 |
+
"grad_norm_max": 1.6858042478561401,
|
| 56 |
+
"clip_fraction": 0.010416666977107525,
|
| 57 |
+
"gpu_peak_gb": 49.363
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.0317767042045792,
|
| 63 |
+
"val_loss": 0.8289197285970052,
|
| 64 |
+
"lr": 0.001,
|
| 65 |
+
"epoch_seconds": 74.1818745136261,
|
| 66 |
+
"train_seconds": 68.86,
|
| 67 |
+
"val_seconds": 5.321,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.7886371314525604,
|
| 72 |
+
"epochs_without_improvement": 2,
|
| 73 |
+
"grad_norm_mean": 0.40695664286613464,
|
| 74 |
+
"grad_norm_max": 1.4001082181930542,
|
| 75 |
+
"clip_fraction": 0.0416666679084301,
|
| 76 |
+
"gpu_peak_gb": 49.363
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.029820364453674603,
|
| 82 |
+
"val_loss": 0.8369706869125366,
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"epoch_seconds": 74.23939228057861,
|
| 85 |
+
"train_seconds": 68.978,
|
| 86 |
+
"val_seconds": 5.261,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.7886371314525604,
|
| 91 |
+
"epochs_without_improvement": 3,
|
| 92 |
+
"grad_norm_mean": 0.2882547378540039,
|
| 93 |
+
"grad_norm_max": 0.8766547441482544,
|
| 94 |
+
"clip_fraction": 0.0,
|
| 95 |
+
"gpu_peak_gb": 49.363
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.030535013414919376,
|
| 101 |
+
"val_loss": 0.8504259139299393,
|
| 102 |
+
"lr": 0.001,
|
| 103 |
+
"epoch_seconds": 74.2548475265503,
|
| 104 |
+
"train_seconds": 68.95,
|
| 105 |
+
"val_seconds": 5.305,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.7886371314525604,
|
| 110 |
+
"epochs_without_improvement": 4,
|
| 111 |
+
"grad_norm_mean": 0.36246535181999207,
|
| 112 |
+
"grad_norm_max": 0.9117569923400879,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 49.363
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.029409046420672286,
|
| 120 |
+
"val_loss": 0.8657974849144617,
|
| 121 |
+
"lr": 0.0005,
|
| 122 |
+
"epoch_seconds": 74.33806705474854,
|
| 123 |
+
"train_seconds": 68.899,
|
| 124 |
+
"val_seconds": 5.439,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.7886371314525604,
|
| 129 |
+
"epochs_without_improvement": 5,
|
| 130 |
+
"grad_norm_mean": 0.28719839453697205,
|
| 131 |
+
"grad_norm_max": 1.1450406312942505,
|
| 132 |
+
"clip_fraction": 0.010416666977107525,
|
| 133 |
+
"gpu_peak_gb": 49.363
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.027889522568633158,
|
| 139 |
+
"val_loss": 0.8729277203480402,
|
| 140 |
+
"lr": 0.0005,
|
| 141 |
+
"epoch_seconds": 74.22603225708008,
|
| 142 |
+
"train_seconds": 68.865,
|
| 143 |
+
"val_seconds": 5.361,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.7886371314525604,
|
| 148 |
+
"epochs_without_improvement": 6,
|
| 149 |
+
"grad_norm_mean": 0.16759946942329407,
|
| 150 |
+
"grad_norm_max": 0.7409897446632385,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 49.363
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.027714470110367984,
|
| 158 |
+
"val_loss": 0.8885419741272926,
|
| 159 |
+
"lr": 0.0005,
|
| 160 |
+
"epoch_seconds": 74.1839964389801,
|
| 161 |
+
"train_seconds": 68.875,
|
| 162 |
+
"val_seconds": 5.309,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.7886371314525604,
|
| 167 |
+
"epochs_without_improvement": 7,
|
| 168 |
+
"grad_norm_mean": 0.1808028668165207,
|
| 169 |
+
"grad_norm_max": 0.8817302584648132,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 49.363
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.027234660209311794,
|
| 177 |
+
"val_loss": 0.891801784435908,
|
| 178 |
+
"lr": 0.0005,
|
| 179 |
+
"epoch_seconds": 74.19446587562561,
|
| 180 |
+
"train_seconds": 68.87,
|
| 181 |
+
"val_seconds": 5.324,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.7886371314525604,
|
| 186 |
+
"epochs_without_improvement": 8,
|
| 187 |
+
"grad_norm_mean": 0.14741063117980957,
|
| 188 |
+
"grad_norm_max": 0.5989755392074585,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 49.363
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.027123544248752296,
|
| 196 |
+
"val_loss": 0.8957383160789808,
|
| 197 |
+
"lr": 0.0005,
|
| 198 |
+
"epoch_seconds": 74.35762929916382,
|
| 199 |
+
"train_seconds": 68.902,
|
| 200 |
+
"val_seconds": 5.456,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.7886371314525604,
|
| 205 |
+
"epochs_without_improvement": 9,
|
| 206 |
+
"grad_norm_mean": 0.1542281210422516,
|
| 207 |
+
"grad_norm_max": 0.6060466766357422,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 49.363
|
| 210 |
+
}
|
| 211 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/logs/transformer_asa_2d.log
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:58:09 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:58:09 - root - INFO - Model parameters: 1,003,374
|
| 3 |
+
2026-09-03 03:58:09 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 03:59:25 - root - INFO - Epoch 0 [train] - train_loss: 0.087173 | val_loss: 0.788637 | lr: 0.001000 | epoch_seconds: 75.990085 | train_seconds: 70.732000 | val_seconds: 5.258000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 0.000000 | grad_norm_mean: 1.955022 | grad_norm_max: 30.717197 | clip_fraction: 0.562500 | gpu_peak_gb: 49.363000
|
| 5 |
+
2026-09-03 04:00:40 - root - INFO - Epoch 1 [train] - train_loss: 0.037517 | val_loss: 0.804539 | lr: 0.001000 | epoch_seconds: 74.849554 | train_seconds: 69.383000 | val_seconds: 5.466000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.615605 | grad_norm_max: 2.214387 | clip_fraction: 0.104167 | gpu_peak_gb: 49.363000
|
| 6 |
+
2026-09-03 04:01:55 - root - INFO - Epoch 2 [train] - train_loss: 0.032341 | val_loss: 0.805234 | lr: 0.001000 | epoch_seconds: 74.449460 | train_seconds: 69.034000 | val_seconds: 5.415000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.355096 | grad_norm_max: 1.685804 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 7 |
+
2026-09-03 04:03:09 - root - INFO - Epoch 3 [train] - train_loss: 0.031777 | val_loss: 0.828920 | lr: 0.001000 | epoch_seconds: 74.181875 | train_seconds: 68.860000 | val_seconds: 5.321000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.406957 | grad_norm_max: 1.400108 | clip_fraction: 0.041667 | gpu_peak_gb: 49.363000
|
| 8 |
+
2026-09-03 04:04:23 - root - INFO - Epoch 4 [train] - train_loss: 0.029820 | val_loss: 0.836971 | lr: 0.001000 | epoch_seconds: 74.239392 | train_seconds: 68.978000 | val_seconds: 5.261000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.288255 | grad_norm_max: 0.876655 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 9 |
+
2026-09-03 04:05:37 - root - INFO - Epoch 5 [train] - train_loss: 0.030535 | val_loss: 0.850426 | lr: 0.001000 | epoch_seconds: 74.254848 | train_seconds: 68.950000 | val_seconds: 5.305000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.362465 | grad_norm_max: 0.911757 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 10 |
+
2026-09-03 04:06:52 - root - INFO - Epoch 6 [train] - train_loss: 0.029409 | val_loss: 0.865797 | lr: 0.000500 | epoch_seconds: 74.338067 | train_seconds: 68.899000 | val_seconds: 5.439000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.287198 | grad_norm_max: 1.145041 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 11 |
+
2026-09-03 04:08:06 - root - INFO - Epoch 7 [train] - train_loss: 0.027890 | val_loss: 0.872928 | lr: 0.000500 | epoch_seconds: 74.226032 | train_seconds: 68.865000 | val_seconds: 5.361000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.167599 | grad_norm_max: 0.740990 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 12 |
+
2026-09-03 04:09:20 - root - INFO - Epoch 8 [train] - train_loss: 0.027714 | val_loss: 0.888542 | lr: 0.000500 | epoch_seconds: 74.183996 | train_seconds: 68.875000 | val_seconds: 5.309000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.180803 | grad_norm_max: 0.881730 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 13 |
+
2026-09-03 04:10:34 - root - INFO - Epoch 9 [train] - train_loss: 0.027235 | val_loss: 0.891802 | lr: 0.000500 | epoch_seconds: 74.194466 | train_seconds: 68.870000 | val_seconds: 5.324000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.147411 | grad_norm_max: 0.598976 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 14 |
+
2026-09-03 04:11:49 - root - INFO - Epoch 10 [train] - train_loss: 0.027124 | val_loss: 0.895738 | lr: 0.000500 | epoch_seconds: 74.357629 | train_seconds: 68.902000 | val_seconds: 5.456000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.788637 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.154228 | grad_norm_max: 0.606047 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 15 |
+
2026-09-03 04:11:49 - root - INFO - Early stopping triggered at epoch 10
|
| 16 |
+
2026-09-03 04:11:49 - root - INFO - Training completed in 819.28 seconds
|
| 17 |
+
2026-09-03 04:11:49 - root - INFO - Realized optimizer steps: 1056 (96/epoch x 11 epochs) at effective batch 30087 observations
|
| 18 |
+
2026-09-03 04:11:50 - root - INFO - Cost [measured]: 6365.74 TFLOP total (fwd+bwd); 578.70 TFLOP to best (epoch 1)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s619/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_asa_2d_id_embeddings_s619",
|
| 3 |
+
"fingerprint": "ccfb7b2734c56fe47f99f29e3bef86e749a82a4af64eb4f1d3def47e24da0292",
|
| 4 |
+
"checkpoint_sha256": "74efe6fa3803d496fd8fd67e38f213f16bf843a9b7abde976728a1b182812c90"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e2dd608ccf9105cbc64dd035d2acb7e0ddd28904e670e1925735a478bffcacb
|
| 3 |
+
size 5089325
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 6028167585024,
|
| 7 |
+
"flops_per_observation": 200357881.6440323,
|
| 8 |
+
"epochs_run": 13,
|
| 9 |
+
"epochs_to_best": 3,
|
| 10 |
+
"optimizer_steps_total": 1248,
|
| 11 |
+
"optimizer_steps_to_best": 288,
|
| 12 |
+
"observations_total": 37548576,
|
| 13 |
+
"observations_to_best": 8665056,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 7523153146109952.0,
|
| 16 |
+
"train_flops_to_best": 1736112264486912.0,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "asa_2d",
|
| 20 |
+
"seed": 732
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/logs/metrics.json
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.09086599969305098,
|
| 6 |
+
"val_loss": 0.8909579540292422,
|
| 7 |
+
"lr": 0.001,
|
| 8 |
+
"epoch_seconds": 76.47002935409546,
|
| 9 |
+
"train_seconds": 70.916,
|
| 10 |
+
"val_seconds": 5.554,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.8909579540292422,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 1.906204104423523,
|
| 17 |
+
"grad_norm_max": 18.91358184814453,
|
| 18 |
+
"clip_fraction": 0.46875,
|
| 19 |
+
"gpu_peak_gb": 49.363
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.036332153889816254,
|
| 25 |
+
"val_loss": 0.8771094307303429,
|
| 26 |
+
"lr": 0.001,
|
| 27 |
+
"epoch_seconds": 74.57579898834229,
|
| 28 |
+
"train_seconds": 69.044,
|
| 29 |
+
"val_seconds": 5.532,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.8771094307303429,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.47463059425354004,
|
| 36 |
+
"grad_norm_max": 1.3598157167434692,
|
| 37 |
+
"clip_fraction": 0.0520833358168602,
|
| 38 |
+
"gpu_peak_gb": 49.363
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.03240994443573678,
|
| 44 |
+
"val_loss": 0.8765473738312721,
|
| 45 |
+
"lr": 0.001,
|
| 46 |
+
"epoch_seconds": 74.47310972213745,
|
| 47 |
+
"train_seconds": 69.015,
|
| 48 |
+
"val_seconds": 5.458,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.8765473738312721,
|
| 53 |
+
"epochs_without_improvement": 0,
|
| 54 |
+
"grad_norm_mean": 0.3450954556465149,
|
| 55 |
+
"grad_norm_max": 1.0513862371444702,
|
| 56 |
+
"clip_fraction": 0.010416666977107525,
|
| 57 |
+
"gpu_peak_gb": 49.363
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.031018618746505428,
|
| 63 |
+
"val_loss": 0.8816855450471243,
|
| 64 |
+
"lr": 0.001,
|
| 65 |
+
"epoch_seconds": 74.4855444431305,
|
| 66 |
+
"train_seconds": 69.056,
|
| 67 |
+
"val_seconds": 5.429,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.8765473738312721,
|
| 72 |
+
"epochs_without_improvement": 0,
|
| 73 |
+
"grad_norm_mean": 0.27852001786231995,
|
| 74 |
+
"grad_norm_max": 0.7958593368530273,
|
| 75 |
+
"clip_fraction": 0.0,
|
| 76 |
+
"gpu_peak_gb": 49.363
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.030296307891452063,
|
| 82 |
+
"val_loss": 0.8887335682908694,
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"epoch_seconds": 74.6505994796753,
|
| 85 |
+
"train_seconds": 69.133,
|
| 86 |
+
"val_seconds": 5.518,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.8765473738312721,
|
| 91 |
+
"epochs_without_improvement": 1,
|
| 92 |
+
"grad_norm_mean": 0.25123685598373413,
|
| 93 |
+
"grad_norm_max": 1.107618808746338,
|
| 94 |
+
"clip_fraction": 0.010416666977107525,
|
| 95 |
+
"gpu_peak_gb": 49.363
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.028706966336661328,
|
| 101 |
+
"val_loss": 0.8885841195782026,
|
| 102 |
+
"lr": 0.001,
|
| 103 |
+
"epoch_seconds": 74.43450355529785,
|
| 104 |
+
"train_seconds": 69.03,
|
| 105 |
+
"val_seconds": 5.404,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.8765473738312721,
|
| 110 |
+
"epochs_without_improvement": 2,
|
| 111 |
+
"grad_norm_mean": 0.18540410697460175,
|
| 112 |
+
"grad_norm_max": 0.8494186401367188,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 49.363
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.028030616173055023,
|
| 120 |
+
"val_loss": 0.8892562836408615,
|
| 121 |
+
"lr": 0.001,
|
| 122 |
+
"epoch_seconds": 74.64552235603333,
|
| 123 |
+
"train_seconds": 68.996,
|
| 124 |
+
"val_seconds": 5.65,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.8765473738312721,
|
| 129 |
+
"epochs_without_improvement": 3,
|
| 130 |
+
"grad_norm_mean": 0.17958585917949677,
|
| 131 |
+
"grad_norm_max": 0.5055558085441589,
|
| 132 |
+
"clip_fraction": 0.0,
|
| 133 |
+
"gpu_peak_gb": 49.363
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.028063227927001815,
|
| 139 |
+
"val_loss": 0.8913777520259222,
|
| 140 |
+
"lr": 0.001,
|
| 141 |
+
"epoch_seconds": 74.70635676383972,
|
| 142 |
+
"train_seconds": 69.189,
|
| 143 |
+
"val_seconds": 5.517,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.8765473738312721,
|
| 148 |
+
"epochs_without_improvement": 4,
|
| 149 |
+
"grad_norm_mean": 0.21726951003074646,
|
| 150 |
+
"grad_norm_max": 0.6755824685096741,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 49.363
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.02726021296499918,
|
| 158 |
+
"val_loss": 0.8893471211194992,
|
| 159 |
+
"lr": 0.0005,
|
| 160 |
+
"epoch_seconds": 74.44259095191956,
|
| 161 |
+
"train_seconds": 68.997,
|
| 162 |
+
"val_seconds": 5.445,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.8765473738312721,
|
| 167 |
+
"epochs_without_improvement": 5,
|
| 168 |
+
"grad_norm_mean": 0.1692858338356018,
|
| 169 |
+
"grad_norm_max": 0.5027985572814941,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 49.363
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.02654520202971374,
|
| 177 |
+
"val_loss": 0.8872716724872589,
|
| 178 |
+
"lr": 0.0005,
|
| 179 |
+
"epoch_seconds": 74.5515513420105,
|
| 180 |
+
"train_seconds": 69.051,
|
| 181 |
+
"val_seconds": 5.5,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.8765473738312721,
|
| 186 |
+
"epochs_without_improvement": 6,
|
| 187 |
+
"grad_norm_mean": 0.1383160948753357,
|
| 188 |
+
"grad_norm_max": 0.41256386041641235,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 49.363
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.026302556158043444,
|
| 196 |
+
"val_loss": 0.8835422818859419,
|
| 197 |
+
"lr": 0.0005,
|
| 198 |
+
"epoch_seconds": 74.40244483947754,
|
| 199 |
+
"train_seconds": 68.964,
|
| 200 |
+
"val_seconds": 5.438,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.8765473738312721,
|
| 205 |
+
"epochs_without_improvement": 7,
|
| 206 |
+
"grad_norm_mean": 0.1267050802707672,
|
| 207 |
+
"grad_norm_max": 0.5300188660621643,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 49.363
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"epoch": 11,
|
| 213 |
+
"phase": "train",
|
| 214 |
+
"train_loss": 0.025964073739790667,
|
| 215 |
+
"val_loss": 0.8817680676778158,
|
| 216 |
+
"lr": 0.0005,
|
| 217 |
+
"epoch_seconds": 74.5636658668518,
|
| 218 |
+
"train_seconds": 69.077,
|
| 219 |
+
"val_seconds": 5.487,
|
| 220 |
+
"steps": 96,
|
| 221 |
+
"total_steps": 1152,
|
| 222 |
+
"observations": 34660224,
|
| 223 |
+
"best_val_loss": 0.8765473738312721,
|
| 224 |
+
"epochs_without_improvement": 8,
|
| 225 |
+
"grad_norm_mean": 0.10478866100311279,
|
| 226 |
+
"grad_norm_max": 0.39568576216697693,
|
| 227 |
+
"clip_fraction": 0.0,
|
| 228 |
+
"gpu_peak_gb": 49.363
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"epoch": 12,
|
| 232 |
+
"phase": "train",
|
| 233 |
+
"train_loss": 0.026037062498896073,
|
| 234 |
+
"val_loss": 0.8783866340915362,
|
| 235 |
+
"lr": 0.0005,
|
| 236 |
+
"epoch_seconds": 74.62879180908203,
|
| 237 |
+
"train_seconds": 69.075,
|
| 238 |
+
"val_seconds": 5.554,
|
| 239 |
+
"steps": 96,
|
| 240 |
+
"total_steps": 1248,
|
| 241 |
+
"observations": 37548576,
|
| 242 |
+
"best_val_loss": 0.8765473738312721,
|
| 243 |
+
"epochs_without_improvement": 9,
|
| 244 |
+
"grad_norm_mean": 0.12291331589221954,
|
| 245 |
+
"grad_norm_max": 0.4887891113758087,
|
| 246 |
+
"clip_fraction": 0.0,
|
| 247 |
+
"gpu_peak_gb": 49.363
|
| 248 |
+
}
|
| 249 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/logs/transformer_asa_2d.log
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:58:10 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:58:10 - root - INFO - Model parameters: 1,003,374
|
| 3 |
+
2026-09-03 03:58:10 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 11:29:37 - root - INFO - Trainer initialized on device: cuda
|
| 5 |
+
2026-09-03 11:29:37 - root - INFO - Model parameters: 1,003,374
|
| 6 |
+
2026-09-03 11:29:37 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 7 |
+
2026-09-03 11:30:53 - root - INFO - Epoch 0 [train] - train_loss: 0.090866 | val_loss: 0.890958 | lr: 0.001000 | epoch_seconds: 76.470029 | train_seconds: 70.916000 | val_seconds: 5.554000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.890958 | epochs_without_improvement: 0.000000 | grad_norm_mean: 1.906204 | grad_norm_max: 18.913582 | clip_fraction: 0.468750 | gpu_peak_gb: 49.363000
|
| 8 |
+
2026-09-03 11:32:08 - root - INFO - Epoch 1 [train] - train_loss: 0.036332 | val_loss: 0.877109 | lr: 0.001000 | epoch_seconds: 74.575799 | train_seconds: 69.044000 | val_seconds: 5.532000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.877109 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.474631 | grad_norm_max: 1.359816 | clip_fraction: 0.052083 | gpu_peak_gb: 49.363000
|
| 9 |
+
2026-09-03 11:33:22 - root - INFO - Epoch 2 [train] - train_loss: 0.032410 | val_loss: 0.876547 | lr: 0.001000 | epoch_seconds: 74.473110 | train_seconds: 69.015000 | val_seconds: 5.458000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.345095 | grad_norm_max: 1.051386 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 10 |
+
2026-09-03 11:34:37 - root - INFO - Epoch 3 [train] - train_loss: 0.031019 | val_loss: 0.881686 | lr: 0.001000 | epoch_seconds: 74.485544 | train_seconds: 69.056000 | val_seconds: 5.429000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.278520 | grad_norm_max: 0.795859 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 11 |
+
2026-09-03 11:35:51 - root - INFO - Epoch 4 [train] - train_loss: 0.030296 | val_loss: 0.888734 | lr: 0.001000 | epoch_seconds: 74.650599 | train_seconds: 69.133000 | val_seconds: 5.518000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.251237 | grad_norm_max: 1.107619 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 12 |
+
2026-09-03 11:37:06 - root - INFO - Epoch 5 [train] - train_loss: 0.028707 | val_loss: 0.888584 | lr: 0.001000 | epoch_seconds: 74.434504 | train_seconds: 69.030000 | val_seconds: 5.404000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.185404 | grad_norm_max: 0.849419 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 13 |
+
2026-09-03 11:38:20 - root - INFO - Epoch 6 [train] - train_loss: 0.028031 | val_loss: 0.889256 | lr: 0.001000 | epoch_seconds: 74.645522 | train_seconds: 68.996000 | val_seconds: 5.650000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.179586 | grad_norm_max: 0.505556 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 14 |
+
2026-09-03 11:39:35 - root - INFO - Epoch 7 [train] - train_loss: 0.028063 | val_loss: 0.891378 | lr: 0.001000 | epoch_seconds: 74.706357 | train_seconds: 69.189000 | val_seconds: 5.517000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.217270 | grad_norm_max: 0.675582 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 15 |
+
2026-09-03 11:40:50 - root - INFO - Epoch 8 [train] - train_loss: 0.027260 | val_loss: 0.889347 | lr: 0.000500 | epoch_seconds: 74.442591 | train_seconds: 68.997000 | val_seconds: 5.445000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.169286 | grad_norm_max: 0.502799 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 16 |
+
2026-09-03 11:42:04 - root - INFO - Epoch 9 [train] - train_loss: 0.026545 | val_loss: 0.887272 | lr: 0.000500 | epoch_seconds: 74.551551 | train_seconds: 69.051000 | val_seconds: 5.500000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.138316 | grad_norm_max: 0.412564 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 17 |
+
2026-09-03 11:43:18 - root - INFO - Epoch 10 [train] - train_loss: 0.026303 | val_loss: 0.883542 | lr: 0.000500 | epoch_seconds: 74.402445 | train_seconds: 68.964000 | val_seconds: 5.438000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.126705 | grad_norm_max: 0.530019 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 18 |
+
2026-09-03 11:44:33 - root - INFO - Epoch 11 [train] - train_loss: 0.025964 | val_loss: 0.881768 | lr: 0.000500 | epoch_seconds: 74.563666 | train_seconds: 69.077000 | val_seconds: 5.487000 | steps: 96.000000 | total_steps: 1152.000000 | observations: 34660224.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.104789 | grad_norm_max: 0.395686 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 19 |
+
2026-09-03 11:45:48 - root - INFO - Epoch 12 [train] - train_loss: 0.026037 | val_loss: 0.878387 | lr: 0.000500 | epoch_seconds: 74.628792 | train_seconds: 69.075000 | val_seconds: 5.554000 | steps: 96.000000 | total_steps: 1248.000000 | observations: 37548576.000000 | best_val_loss: 0.876547 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.122913 | grad_norm_max: 0.488789 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 20 |
+
2026-09-03 11:45:48 - root - INFO - Early stopping triggered at epoch 12
|
| 21 |
+
2026-09-03 11:45:48 - root - INFO - Training completed in 971.07 seconds
|
| 22 |
+
2026-09-03 11:45:48 - root - INFO - Realized optimizer steps: 1248 (96/epoch x 13 epochs) at effective batch 30087 observations
|
| 23 |
+
2026-09-03 11:45:49 - root - INFO - Cost [measured]: 7523.15 TFLOP total (fwd+bwd); 1736.11 TFLOP to best (epoch 3)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s732/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_asa_2d_id_embeddings_s732",
|
| 3 |
+
"fingerprint": "6d827d9862f8a1888d435eee32bd6fb8251e1162eb974c542211fafbd3e700c2",
|
| 4 |
+
"checkpoint_sha256": "9e2dd608ccf9105cbc64dd035d2acb7e0ddd28904e670e1925735a478bffcacb"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/best.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2185e2fda873c0e678ff23274e5b3b12e85db32f3a1d546c7df2cd4a96b5860c
|
| 3 |
+
size 5089325
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/cost.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"status": "measured",
|
| 3 |
+
"flops_uncounted_modules": [],
|
| 4 |
+
"micro_batches_measured": 1,
|
| 5 |
+
"observations_measured": 30087,
|
| 6 |
+
"flops_per_optimizer_step": 6028167585024,
|
| 7 |
+
"flops_per_observation": 200357881.6440323,
|
| 8 |
+
"epochs_run": 11,
|
| 9 |
+
"epochs_to_best": 1,
|
| 10 |
+
"optimizer_steps_total": 1056,
|
| 11 |
+
"optimizer_steps_to_best": 96,
|
| 12 |
+
"observations_total": 31771872,
|
| 13 |
+
"observations_to_best": 2888352,
|
| 14 |
+
"effective_batch_size": 30087,
|
| 15 |
+
"train_flops_total": 6365744969785344.0,
|
| 16 |
+
"train_flops_to_best": 578704088162304.0,
|
| 17 |
+
"validation_flops": null,
|
| 18 |
+
"model": "transformer",
|
| 19 |
+
"variant": "asa_2d",
|
| 20 |
+
"seed": 947
|
| 21 |
+
}
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/logs/metrics.json
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"epoch": 0,
|
| 4 |
+
"phase": "train",
|
| 5 |
+
"train_loss": 0.09446391618500154,
|
| 6 |
+
"val_loss": 0.8052720750371615,
|
| 7 |
+
"lr": 0.001,
|
| 8 |
+
"epoch_seconds": 77.06520628929138,
|
| 9 |
+
"train_seconds": 71.62,
|
| 10 |
+
"val_seconds": 5.445,
|
| 11 |
+
"steps": 96,
|
| 12 |
+
"total_steps": 96,
|
| 13 |
+
"observations": 2888352,
|
| 14 |
+
"best_val_loss": 0.8052720750371615,
|
| 15 |
+
"epochs_without_improvement": 0,
|
| 16 |
+
"grad_norm_mean": 2.0476620197296143,
|
| 17 |
+
"grad_norm_max": 42.00563049316406,
|
| 18 |
+
"clip_fraction": 0.4583333432674408,
|
| 19 |
+
"gpu_peak_gb": 49.363
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"epoch": 1,
|
| 23 |
+
"phase": "train",
|
| 24 |
+
"train_loss": 0.03563512690986196,
|
| 25 |
+
"val_loss": 0.8228440831104914,
|
| 26 |
+
"lr": 0.001,
|
| 27 |
+
"epoch_seconds": 75.26004791259766,
|
| 28 |
+
"train_seconds": 69.725,
|
| 29 |
+
"val_seconds": 5.535,
|
| 30 |
+
"steps": 96,
|
| 31 |
+
"total_steps": 192,
|
| 32 |
+
"observations": 5776704,
|
| 33 |
+
"best_val_loss": 0.8052720750371615,
|
| 34 |
+
"epochs_without_improvement": 0,
|
| 35 |
+
"grad_norm_mean": 0.5059750080108643,
|
| 36 |
+
"grad_norm_max": 1.507473349571228,
|
| 37 |
+
"clip_fraction": 0.1041666716337204,
|
| 38 |
+
"gpu_peak_gb": 49.363
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"epoch": 2,
|
| 42 |
+
"phase": "train",
|
| 43 |
+
"train_loss": 0.03327810042537749,
|
| 44 |
+
"val_loss": 0.825155052045981,
|
| 45 |
+
"lr": 0.001,
|
| 46 |
+
"epoch_seconds": 75.28269267082214,
|
| 47 |
+
"train_seconds": 69.58,
|
| 48 |
+
"val_seconds": 5.702,
|
| 49 |
+
"steps": 96,
|
| 50 |
+
"total_steps": 288,
|
| 51 |
+
"observations": 8665056,
|
| 52 |
+
"best_val_loss": 0.8052720750371615,
|
| 53 |
+
"epochs_without_improvement": 1,
|
| 54 |
+
"grad_norm_mean": 0.46359214186668396,
|
| 55 |
+
"grad_norm_max": 1.5445765256881714,
|
| 56 |
+
"clip_fraction": 0.0416666679084301,
|
| 57 |
+
"gpu_peak_gb": 49.363
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"epoch": 3,
|
| 61 |
+
"phase": "train",
|
| 62 |
+
"train_loss": 0.03141009889077395,
|
| 63 |
+
"val_loss": 0.8225759168465933,
|
| 64 |
+
"lr": 0.001,
|
| 65 |
+
"epoch_seconds": 75.02649807929993,
|
| 66 |
+
"train_seconds": 69.546,
|
| 67 |
+
"val_seconds": 5.481,
|
| 68 |
+
"steps": 96,
|
| 69 |
+
"total_steps": 384,
|
| 70 |
+
"observations": 11553408,
|
| 71 |
+
"best_val_loss": 0.8052720750371615,
|
| 72 |
+
"epochs_without_improvement": 2,
|
| 73 |
+
"grad_norm_mean": 0.3471825122833252,
|
| 74 |
+
"grad_norm_max": 1.2014389038085938,
|
| 75 |
+
"clip_fraction": 0.02083333395421505,
|
| 76 |
+
"gpu_peak_gb": 49.363
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"epoch": 4,
|
| 80 |
+
"phase": "train",
|
| 81 |
+
"train_loss": 0.030225392935487132,
|
| 82 |
+
"val_loss": 0.8227396284540495,
|
| 83 |
+
"lr": 0.001,
|
| 84 |
+
"epoch_seconds": 75.1506974697113,
|
| 85 |
+
"train_seconds": 69.635,
|
| 86 |
+
"val_seconds": 5.515,
|
| 87 |
+
"steps": 96,
|
| 88 |
+
"total_steps": 480,
|
| 89 |
+
"observations": 14441760,
|
| 90 |
+
"best_val_loss": 0.8052720750371615,
|
| 91 |
+
"epochs_without_improvement": 3,
|
| 92 |
+
"grad_norm_mean": 0.2997567057609558,
|
| 93 |
+
"grad_norm_max": 1.329148530960083,
|
| 94 |
+
"clip_fraction": 0.010416666977107525,
|
| 95 |
+
"gpu_peak_gb": 49.363
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"epoch": 5,
|
| 99 |
+
"phase": "train",
|
| 100 |
+
"train_loss": 0.029317451873794198,
|
| 101 |
+
"val_loss": 0.8351333513855934,
|
| 102 |
+
"lr": 0.001,
|
| 103 |
+
"epoch_seconds": 75.0349612236023,
|
| 104 |
+
"train_seconds": 69.566,
|
| 105 |
+
"val_seconds": 5.469,
|
| 106 |
+
"steps": 96,
|
| 107 |
+
"total_steps": 576,
|
| 108 |
+
"observations": 17330112,
|
| 109 |
+
"best_val_loss": 0.8052720750371615,
|
| 110 |
+
"epochs_without_improvement": 4,
|
| 111 |
+
"grad_norm_mean": 0.2606029212474823,
|
| 112 |
+
"grad_norm_max": 0.7720146775245667,
|
| 113 |
+
"clip_fraction": 0.0,
|
| 114 |
+
"gpu_peak_gb": 49.363
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"epoch": 6,
|
| 118 |
+
"phase": "train",
|
| 119 |
+
"train_loss": 0.029723857024994988,
|
| 120 |
+
"val_loss": 0.8656105374296507,
|
| 121 |
+
"lr": 0.0005,
|
| 122 |
+
"epoch_seconds": 75.24637794494629,
|
| 123 |
+
"train_seconds": 69.741,
|
| 124 |
+
"val_seconds": 5.506,
|
| 125 |
+
"steps": 96,
|
| 126 |
+
"total_steps": 672,
|
| 127 |
+
"observations": 20218464,
|
| 128 |
+
"best_val_loss": 0.8052720750371615,
|
| 129 |
+
"epochs_without_improvement": 5,
|
| 130 |
+
"grad_norm_mean": 0.315094530582428,
|
| 131 |
+
"grad_norm_max": 1.2380321025848389,
|
| 132 |
+
"clip_fraction": 0.010416666977107525,
|
| 133 |
+
"gpu_peak_gb": 49.363
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"epoch": 7,
|
| 137 |
+
"phase": "train",
|
| 138 |
+
"train_loss": 0.028098967062154163,
|
| 139 |
+
"val_loss": 0.8594051425655683,
|
| 140 |
+
"lr": 0.0005,
|
| 141 |
+
"epoch_seconds": 75.14561748504639,
|
| 142 |
+
"train_seconds": 69.658,
|
| 143 |
+
"val_seconds": 5.488,
|
| 144 |
+
"steps": 96,
|
| 145 |
+
"total_steps": 768,
|
| 146 |
+
"observations": 23106816,
|
| 147 |
+
"best_val_loss": 0.8052720750371615,
|
| 148 |
+
"epochs_without_improvement": 6,
|
| 149 |
+
"grad_norm_mean": 0.18055610358715057,
|
| 150 |
+
"grad_norm_max": 0.8388330340385437,
|
| 151 |
+
"clip_fraction": 0.0,
|
| 152 |
+
"gpu_peak_gb": 49.363
|
| 153 |
+
},
|
| 154 |
+
{
|
| 155 |
+
"epoch": 8,
|
| 156 |
+
"phase": "train",
|
| 157 |
+
"train_loss": 0.02784236763060714,
|
| 158 |
+
"val_loss": 0.8600875288248062,
|
| 159 |
+
"lr": 0.0005,
|
| 160 |
+
"epoch_seconds": 75.10526657104492,
|
| 161 |
+
"train_seconds": 69.519,
|
| 162 |
+
"val_seconds": 5.587,
|
| 163 |
+
"steps": 96,
|
| 164 |
+
"total_steps": 864,
|
| 165 |
+
"observations": 25995168,
|
| 166 |
+
"best_val_loss": 0.8052720750371615,
|
| 167 |
+
"epochs_without_improvement": 7,
|
| 168 |
+
"grad_norm_mean": 0.18868955969810486,
|
| 169 |
+
"grad_norm_max": 0.7405126094818115,
|
| 170 |
+
"clip_fraction": 0.0,
|
| 171 |
+
"gpu_peak_gb": 49.363
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"epoch": 9,
|
| 175 |
+
"phase": "train",
|
| 176 |
+
"train_loss": 0.02763204228055353,
|
| 177 |
+
"val_loss": 0.859373206893603,
|
| 178 |
+
"lr": 0.0005,
|
| 179 |
+
"epoch_seconds": 75.16298246383667,
|
| 180 |
+
"train_seconds": 69.574,
|
| 181 |
+
"val_seconds": 5.589,
|
| 182 |
+
"steps": 96,
|
| 183 |
+
"total_steps": 960,
|
| 184 |
+
"observations": 28883520,
|
| 185 |
+
"best_val_loss": 0.8052720750371615,
|
| 186 |
+
"epochs_without_improvement": 8,
|
| 187 |
+
"grad_norm_mean": 0.1918925940990448,
|
| 188 |
+
"grad_norm_max": 0.49790719151496887,
|
| 189 |
+
"clip_fraction": 0.0,
|
| 190 |
+
"gpu_peak_gb": 49.363
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"epoch": 10,
|
| 194 |
+
"phase": "train",
|
| 195 |
+
"train_loss": 0.027106597806171823,
|
| 196 |
+
"val_loss": 0.8561204150319099,
|
| 197 |
+
"lr": 0.0005,
|
| 198 |
+
"epoch_seconds": 75.1426990032196,
|
| 199 |
+
"train_seconds": 69.575,
|
| 200 |
+
"val_seconds": 5.568,
|
| 201 |
+
"steps": 96,
|
| 202 |
+
"total_steps": 1056,
|
| 203 |
+
"observations": 31771872,
|
| 204 |
+
"best_val_loss": 0.8052720750371615,
|
| 205 |
+
"epochs_without_improvement": 9,
|
| 206 |
+
"grad_norm_mean": 0.1380597949028015,
|
| 207 |
+
"grad_norm_max": 0.5615152716636658,
|
| 208 |
+
"clip_fraction": 0.0,
|
| 209 |
+
"gpu_peak_gb": 49.363
|
| 210 |
+
}
|
| 211 |
+
]
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/logs/transformer_asa_2d.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-09-03 03:58:10 - root - INFO - Trainer initialized on device: cuda
|
| 2 |
+
2026-09-03 03:58:10 - root - INFO - Model parameters: 1,003,374
|
| 3 |
+
2026-09-03 03:58:10 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 4 |
+
2026-09-03 11:29:37 - root - INFO - Trainer initialized on device: cuda
|
| 5 |
+
2026-09-03 11:29:37 - root - INFO - Model parameters: 1,003,374
|
| 6 |
+
2026-09-03 11:29:37 - root - INFO - Batch budget: micro-batch 1 x grad_accum 1 = effective batch 30087 observations | 96 micro-batches/epoch -> 96 optimizer steps/epoch | max 19200 steps over 200 epochs
|
| 7 |
+
2026-09-03 11:30:54 - root - INFO - Epoch 0 [train] - train_loss: 0.094464 | val_loss: 0.805272 | lr: 0.001000 | epoch_seconds: 77.065206 | train_seconds: 71.620000 | val_seconds: 5.445000 | steps: 96.000000 | total_steps: 96.000000 | observations: 2888352.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 0.000000 | grad_norm_mean: 2.047662 | grad_norm_max: 42.005630 | clip_fraction: 0.458333 | gpu_peak_gb: 49.363000
|
| 8 |
+
2026-09-03 11:32:09 - root - INFO - Epoch 1 [train] - train_loss: 0.035635 | val_loss: 0.822844 | lr: 0.001000 | epoch_seconds: 75.260048 | train_seconds: 69.725000 | val_seconds: 5.535000 | steps: 96.000000 | total_steps: 192.000000 | observations: 5776704.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 0.000000 | grad_norm_mean: 0.505975 | grad_norm_max: 1.507473 | clip_fraction: 0.104167 | gpu_peak_gb: 49.363000
|
| 9 |
+
2026-09-03 11:33:24 - root - INFO - Epoch 2 [train] - train_loss: 0.033278 | val_loss: 0.825155 | lr: 0.001000 | epoch_seconds: 75.282693 | train_seconds: 69.580000 | val_seconds: 5.702000 | steps: 96.000000 | total_steps: 288.000000 | observations: 8665056.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 1.000000 | grad_norm_mean: 0.463592 | grad_norm_max: 1.544577 | clip_fraction: 0.041667 | gpu_peak_gb: 49.363000
|
| 10 |
+
2026-09-03 11:34:39 - root - INFO - Epoch 3 [train] - train_loss: 0.031410 | val_loss: 0.822576 | lr: 0.001000 | epoch_seconds: 75.026498 | train_seconds: 69.546000 | val_seconds: 5.481000 | steps: 96.000000 | total_steps: 384.000000 | observations: 11553408.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 2.000000 | grad_norm_mean: 0.347183 | grad_norm_max: 1.201439 | clip_fraction: 0.020833 | gpu_peak_gb: 49.363000
|
| 11 |
+
2026-09-03 11:35:55 - root - INFO - Epoch 4 [train] - train_loss: 0.030225 | val_loss: 0.822740 | lr: 0.001000 | epoch_seconds: 75.150697 | train_seconds: 69.635000 | val_seconds: 5.515000 | steps: 96.000000 | total_steps: 480.000000 | observations: 14441760.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 3.000000 | grad_norm_mean: 0.299757 | grad_norm_max: 1.329149 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 12 |
+
2026-09-03 11:37:10 - root - INFO - Epoch 5 [train] - train_loss: 0.029317 | val_loss: 0.835133 | lr: 0.001000 | epoch_seconds: 75.034961 | train_seconds: 69.566000 | val_seconds: 5.469000 | steps: 96.000000 | total_steps: 576.000000 | observations: 17330112.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 4.000000 | grad_norm_mean: 0.260603 | grad_norm_max: 0.772015 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 13 |
+
2026-09-03 11:38:25 - root - INFO - Epoch 6 [train] - train_loss: 0.029724 | val_loss: 0.865611 | lr: 0.000500 | epoch_seconds: 75.246378 | train_seconds: 69.741000 | val_seconds: 5.506000 | steps: 96.000000 | total_steps: 672.000000 | observations: 20218464.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 5.000000 | grad_norm_mean: 0.315095 | grad_norm_max: 1.238032 | clip_fraction: 0.010417 | gpu_peak_gb: 49.363000
|
| 14 |
+
2026-09-03 11:39:40 - root - INFO - Epoch 7 [train] - train_loss: 0.028099 | val_loss: 0.859405 | lr: 0.000500 | epoch_seconds: 75.145617 | train_seconds: 69.658000 | val_seconds: 5.488000 | steps: 96.000000 | total_steps: 768.000000 | observations: 23106816.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 6.000000 | grad_norm_mean: 0.180556 | grad_norm_max: 0.838833 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 15 |
+
2026-09-03 11:40:55 - root - INFO - Epoch 8 [train] - train_loss: 0.027842 | val_loss: 0.860088 | lr: 0.000500 | epoch_seconds: 75.105267 | train_seconds: 69.519000 | val_seconds: 5.587000 | steps: 96.000000 | total_steps: 864.000000 | observations: 25995168.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 7.000000 | grad_norm_mean: 0.188690 | grad_norm_max: 0.740513 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 16 |
+
2026-09-03 11:42:10 - root - INFO - Epoch 9 [train] - train_loss: 0.027632 | val_loss: 0.859373 | lr: 0.000500 | epoch_seconds: 75.162982 | train_seconds: 69.574000 | val_seconds: 5.589000 | steps: 96.000000 | total_steps: 960.000000 | observations: 28883520.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 8.000000 | grad_norm_mean: 0.191893 | grad_norm_max: 0.497907 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 17 |
+
2026-09-03 11:43:25 - root - INFO - Epoch 10 [train] - train_loss: 0.027107 | val_loss: 0.856120 | lr: 0.000500 | epoch_seconds: 75.142699 | train_seconds: 69.575000 | val_seconds: 5.568000 | steps: 96.000000 | total_steps: 1056.000000 | observations: 31771872.000000 | best_val_loss: 0.805272 | epochs_without_improvement: 9.000000 | grad_norm_mean: 0.138060 | grad_norm_max: 0.561515 | clip_fraction: 0.000000 | gpu_peak_gb: 49.363000
|
| 18 |
+
2026-09-03 11:43:25 - root - INFO - Early stopping triggered at epoch 10
|
| 19 |
+
2026-09-03 11:43:25 - root - INFO - Training completed in 828.64 seconds
|
| 20 |
+
2026-09-03 11:43:25 - root - INFO - Realized optimizer steps: 1056 (96/epoch x 11 epochs) at effective batch 30087 observations
|
| 21 |
+
2026-09-03 11:43:27 - root - INFO - Cost [measured]: 6365.74 TFLOP total (fwd+bwd); 578.70 TFLOP to best (epoch 1)
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/plots/loss_curve.png
ADDED
|
exp7_exp8_operator_study/final/new_runs/sig_backfill_id/transformer_asa_2d_id_embeddings_s947/run_complete.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "transformer_asa_2d_id_embeddings_s947",
|
| 3 |
+
"fingerprint": "60234ceab0ccffc4c33207d8a3d855106dab4c246cb945edb40a7d28aac4e495",
|
| 4 |
+
"checkpoint_sha256": "2185e2fda873c0e678ff23274e5b3b12e85db32f3a1d546c7df2cd4a96b5860c"
|
| 5 |
+
}
|
exp7_exp8_operator_study/final/results/errdump_FINAL.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12e8528fc91d4d97ec0727d3f376df47a817d7e16a371c6799322e9f069aa3da
|
| 3 |
+
size 586878416
|
exp7_exp8_operator_study/final/results/significance_FINAL_v3.txt
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scipy=True
|
| 2 |
+
pair dMSE | DM_sq p | DM_abs p | Wilcx_abs z p | t_sq p
|
| 3 |
+
S4ND-4D vs S4 flat -0.011063 | -16.40 0.000 | -3.55 0.002 | 103.30 0.00e+00 | -14.50 1.80e-47
|
| 4 |
+
S4 flat vs Transformer -0.715325 | -989.69 0.000 | -208.64 0.000 | -114.05 0.00e+00 | -18.47 9.21e-76
|
| 5 |
+
S4 flat vs GRU FA-2D -0.773679 | -241.96 0.000 | -267.88 0.000 | -149.55 0.00e+00 | -18.98 6.88e-80
|
| 6 |
+
FA authors vs ASA 2-D +0.163751 | 47.22 0.000 | 132.91 0.000 | 145.51 0.00e+00 | 37.85 3.08e-306
|
| 7 |
+
identity vs blind (Tr 2D) +0.040906 | 25.47 0.000 | 2.71 0.012 | 1.78 7.55e-02 | 20.60 1.23e-93
|
| 8 |
+
fa_local vs FA authors (GRU 2D) -0.189978 | -54.10 0.000 | -147.75 0.000 | -147.90 0.00e+00 | -36.25 1.36e-281
|
| 9 |
+
fa_sm vs FA authors (GRU 2D) -0.139157 | -42.05 0.000 | -159.39 0.000 | -145.38 0.00e+00 | -42.47 0.00e+00
|
| 10 |
+
fa_local vs fa_sm (GRU 2D) -0.050821 | -228.38 0.000 | -53.10 0.000 | -121.67 0.00e+00 | -25.46 1.92e-141
|
| 11 |
+
fa_local vs ASA (GRU 2D) -0.026228 | -316.96 0.000 | -110.36 0.000 | -105.82 0.00e+00 | -27.87 8.07e-169
|
| 12 |
+
fa_sm vs ASA (GRU 2D) +0.024594 | 125.38 0.000 | 3.81 0.001 | 1.41 1.58e-01 | 22.87 8.08e-115
|
| 13 |
+
fa_local vs ASA (Tr 2D) -0.057393 | -35.70 0.000 | -35.26 0.000 | -129.12 0.00e+00 | -25.09 1.52e-137
|
| 14 |
+
fa_sm vs ASA (Tr 2D) -0.012242 | -25.39 0.000 | -6.38 0.000 | -50.19 0.00e+00 | -24.37 5.80e-130
|
| 15 |
+
fa_local vs fa_sm (Tr 2D) -0.045151 | -22.27 0.000 | -19.91 0.000 | -130.67 0.00e+00 | -20.53 4.92e-93
|
| 16 |
+
fa_local vs ASA (gru 3D) +0.002351 | 28.10 0.000 | -26.66 0.000 | -114.76 0.00e+00 | 9.37 7.45e-21
|
| 17 |
+
fa_local vs ASA (gru 4D) +0.006252 | 34.90 0.000 | -7.31 0.000 | -82.35 0.00e+00 | 17.28 1.43e-66
|
| 18 |
+
fa_local vs ASA (lstm 3D) -0.084532 | -27.74 0.000 | 32.37 0.000 | 128.96 0.00e+00 | -13.87 1.27e-43
|
| 19 |
+
fa_local vs ASA (lstm 4D) -0.102333 | -122.57 0.000 | 14.42 0.000 | 112.47 0.00e+00 | -13.37 1.14e-40
|
| 20 |
+
fa_local vs ASA (transformer 3D) +0.030305 | 22.63 0.000 | -13.35 0.000 | -115.12 0.00e+00 | 17.80 1.51e-70
|
| 21 |
+
fa_local vs ASA (transformer 4D) -0.243892 | -74.26 0.000 | -92.34 0.000 | -149.12 0.00e+00 | -28.96 6.76e-182
|
| 22 |
+
fa vs ASA (gru 3D) +0.132235 | 133.62 0.000 | 139.51 0.000 | 143.62 0.00e+00 | 42.34 0.00e+00
|
| 23 |
+
fa vs ASA (gru 4D) +0.141858 | 70.14 0.000 | 132.77 0.000 | 145.18 0.00e+00 | 36.37 1.97e-283
|
| 24 |
+
fa vs ASA (lstm 3D) +0.011177 | 56.46 0.000 | 1.54 0.137 | -20.58 4.11e-94 | 18.54 2.75e-76
|
| 25 |
+
fa vs ASA (lstm 4D) +0.008626 | 46.60 0.000 | -34.59 0.000 | -114.13 0.00e+00 | 11.66 2.32e-31
|
| 26 |
+
fa vs ASA (transformer 3D) +0.049052 | 34.72 0.000 | -24.49 0.000 | -120.72 0.00e+00 | 18.50 5.24e-76
|
| 27 |
+
fa vs ASA (transformer 4D) -0.145997 | -39.60 0.000 | -35.49 0.000 | -126.13 0.00e+00 | -31.05 2.06e-208
|
| 28 |
+
S4 flat vs persistence +0.007160 | 9.77 0.000 | 0.64 0.529 | -108.93 0.00e+00 | 7.03 2.09e-12
|
| 29 |
+
S4ND-4D vs persistence -0.003902 | -8.30 0.000 | -1.73 0.098 | -74.19 0.00e+00 | -4.42 9.89e-06
|
| 30 |
+
|
| 31 |
+
equivalence on squared-loss DM differentials | delta mode=frac value=0.05 alpha=0.05
|
| 32 |
+
pair d delta | 90% CI | p_notsup p_noninf p_TOST equiv
|
| 33 |
+
S4ND-4D vs S4 flat -0.011063 0.003103 | [-0.01222,-0.00991] | 1.00e+00 8.29e-17 1.00e+00 no
|
| 34 |
+
S4 flat vs Transformer -0.715325 0.038869 | [-0.71656,-0.71409] | 1.00e+00 1.41e-55 1.00e+00 no
|
| 35 |
+
S4 flat vs GRU FA-2D -0.773679 0.041787 | [-0.77916,-0.76820] | 1.00e+00 1.67e-41 1.00e+00 no
|
| 36 |
+
FA authors vs ASA 2-D +0.163751 0.033599 | [+0.15781,+0.16969] | 1.49e-26 1.00e+00 1.00e+00 no
|
| 37 |
+
identity vs blind (Tr 2D) +0.040906 0.036790 | [+0.03815,+0.04366] | 6.04e-25 9.91e-01 9.91e-01 no
|
| 38 |
+
fa_local vs FA authors (GRU 2D) -0.189978 0.041787 | [-0.19600,-0.18396] | 1.00e+00 5.01e-28 1.00e+00 no
|
| 39 |
+
fa_sm vs FA authors (GRU 2D) -0.139157 0.041787 | [-0.14483,-0.13349] | 1.00e+00 3.70e-26 1.00e+00 no
|
| 40 |
+
fa_local vs fa_sm (GRU 2D) -0.050822 0.034829 | [-0.05120,-0.05044] | 1.00e+00 1.29e-45 1.00e+00 no
|
| 41 |
+
fa_local vs ASA (GRU 2D) -0.026228 0.033599 | [-0.02637,-0.02609] | 5.20e-31 6.53e-52 5.20e-31 YES
|
| 42 |
+
fa_sm vs ASA (GRU 2D) +0.024594 0.033599 | [+0.02426,+0.02493] | 5.15e-43 1.99e-24 1.99e-24 YES
|
| 43 |
+
fa_local vs ASA (Tr 2D) -0.057393 0.036790 | [-0.06015,-0.05464] | 1.00e+00 7.65e-27 1.00e+00 no
|
| 44 |
+
fa_sm vs ASA (Tr 2D) -0.012242 0.036790 | [-0.01307,-0.01142] | 1.88e-25 2.49e-32 1.88e-25 YES
|
| 45 |
+
fa_local vs fa_sm (Tr 2D) -0.045151 0.036178 | [-0.04863,-0.04168] | 1.00e+00 4.29e-23 1.00e+00 no
|
| 46 |
+
fa_local vs ASA (gru 3D) +0.002351 0.034058 | [+0.00221,+0.00249] | 7.64e-47 1.84e-45 1.84e-45 YES
|
| 47 |
+
fa_local vs ASA (gru 4D) +0.006252 0.034143 | [+0.00595,+0.00656] | 2.82e-40 1.40e-36 1.40e-36 YES
|
| 48 |
+
fa_local vs ASA (lstm 3D) -0.084532 0.042949 | [-0.08975,-0.07931] | 1.00e+00 1.65e-23 1.00e+00 no
|
| 49 |
+
fa_local vs ASA (lstm 4D) -0.102333 0.042827 | [-0.10376,-0.10090] | 1.00e+00 1.11e-37 1.00e+00 no
|
| 50 |
+
fa_local vs ASA (transformer 3D) +0.030305 0.035667 | [+0.02801,+0.03260] | 4.01e-25 2.79e-04 2.79e-04 YES
|
| 51 |
+
fa_local vs ASA (transformer 4D) -0.243892 0.046685 | [-0.24952,-0.23826] | 1.00e+00 6.09e-31 1.00e+00 no
|
| 52 |
+
fa vs ASA (gru 3D) +0.132235 0.034058 | [+0.13054,+0.13393] | 2.44e-37 1.00e+00 1.00e+00 no
|
| 53 |
+
fa vs ASA (gru 4D) +0.141858 0.034143 | [+0.13839,+0.14532] | 8.90e-31 1.00e+00 1.00e+00 no
|
| 54 |
+
fa vs ASA (lstm 3D) +0.011177 0.042949 | [+0.01084,+0.01152] | 3.37e-42 7.01e-37 7.01e-37 YES
|
| 55 |
+
fa vs ASA (lstm 4D) +0.008626 0.042827 | [+0.00831,+0.00894] | 2.30e-42 2.75e-38 2.75e-38 YES
|
| 56 |
+
fa vs ASA (transformer 3D) +0.049052 0.035667 | [+0.04663,+0.05147] | 4.51e-27 1.00e+00 1.00e+00 no
|
| 57 |
+
fa vs ASA (transformer 4D) -0.145997 0.046685 | [-0.15232,-0.13968] | 1.00e+00 1.04e-25 1.00e+00 no
|
| 58 |
+
S4 flat vs persistence +0.007160 0.002745 | [+0.00590,+0.00842] | 9.86e-13 1.00e+00 1.00e+00 no
|
| 59 |
+
S4ND-4D vs persistence -0.003902 0.002745 | [-0.00471,-0.00310] | 9.89e-01 3.89e-13 9.89e-01 no
|
| 60 |
+
|
| 61 |
+
d = lossA - lossB (negative => A better). p_notsup small => A is NOT
|
| 62 |
+
better than B by delta; p_TOST small => equivalent within +/-delta.
|
| 63 |
+
A large p is never evidence of equivalence -- read p_TOST, not DM p.
|