EPFL ALiBi and training provenance 901-950 of 1184
Browse files- epfl_final/train_logs/365m_edge_dense_altmixed_seed49.log +445 -0
- epfl_final/train_logs/365m_edge_dense_altmixed_seed49_resume.log +148 -0
- epfl_final/train_logs/a100_40gb_vram_probe_p28.log +167 -0
- epfl_final/train_logs/dense_dropout_p28_8k365m_s43.run.log +264 -0
- epfl_final/train_logs/dense_dropout_p28_8k365m_s44.run.log +264 -0
- epfl_final/train_logs/dense_dropout_p28_8k365m_s45.run.log +264 -0
- epfl_final/train_logs/dense_dropout_p28_8k365m_s46.run.log +264 -0
- epfl_final/train_logs/eval_seed71_dense_50462_quick.log +64 -0
- epfl_final/train_logs/launch_seed72_mixed_after_dense70.log +0 -0
- epfl_final/train_logs/launch_seed72_mixed_after_dense70.pid +1 -0
- epfl_final/train_logs/launch_seed72_mixed_after_dense70.sh +8 -0
- epfl_final/train_logs/p0_frozen_matched_20260821.log +1 -0
- epfl_final/train_logs/p0_frozen_matched_20260821.pid +1 -0
- epfl_final/train_logs/p0_frozen_matched_20260821_resume.log +1 -0
- epfl_final/train_logs/p0_frozen_matched_20260821_resume.pid +1 -0
- epfl_final/train_logs/p0_frozen_matched_a100_20260821.log +1 -0
- epfl_final/train_logs/p0_frozen_matched_a100_20260821.pid +1 -0
- epfl_final/train_logs/p1_control_eval_scheduler_20260822.log +98 -0
- epfl_final/train_logs/p1_control_eval_scheduler_20260823.log +2 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu.log +295 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_relaunch.log +893 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume2.log +383 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume3.log +686 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume_latest.log +395 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu.log +177 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu_a100_after_eval.log +749 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu_h1004.log +223 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_5gpu.log +806 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_5gpu_relaunch.log +561 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_6gpu_resume41750.log +1186 -0
- epfl_final/train_logs/p2_1p2b_dense_seed70_6gpu_switch.log +1154 -0
- epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu.log +476 -0
- epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume.log +0 -0
- epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume2.log +0 -0
- epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume_latest.log +723 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_2gpu.log +543 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch.log +1024 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch2.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch3.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_resume48750.log +280 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed71_5gpu_autoresume.log +210 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_autoresume.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_swapped.log +286 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_swapped.log +362 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.log +0 -0
- epfl_final/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.pid +1 -0
- epfl_final/train_logs/seed70_mixed_48962_battery.log +78 -0
- epfl_final/train_logs/seed70_mixed_48962_battery_a10080.log +13 -0
epfl_final/train_logs/365m_edge_dense_altmixed_seed49.log
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725
|
| 2 |
+
Running config:
|
| 3 |
+
{
|
| 4 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 5 |
+
"model_dtype": "bfloat16",
|
| 6 |
+
"init_from_scratch": true,
|
| 7 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 8 |
+
"val_path": null,
|
| 9 |
+
"max_seq_length": 8192,
|
| 10 |
+
"num_workers": 8,
|
| 11 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725",
|
| 12 |
+
"resume_from_checkpoint": null,
|
| 13 |
+
"force_new_run": true,
|
| 14 |
+
"num_epochs": 1,
|
| 15 |
+
"max_steps": 44000,
|
| 16 |
+
"per_device_batch_size": 2,
|
| 17 |
+
"gradient_accumulation_steps": 10,
|
| 18 |
+
"learning_rate": 0.0005,
|
| 19 |
+
"weight_decay": 0.01,
|
| 20 |
+
"warmup_steps": 750,
|
| 21 |
+
"lr_scheduler": "cosine",
|
| 22 |
+
"max_grad_norm": 1.0,
|
| 23 |
+
"save_steps": 250,
|
| 24 |
+
"eval_steps": 250,
|
| 25 |
+
"eval_batches": 50,
|
| 26 |
+
"logging_steps": 10,
|
| 27 |
+
"seed": 49,
|
| 28 |
+
"verbose": false,
|
| 29 |
+
"checkpoint_activations": false,
|
| 30 |
+
"strategy": "ddp",
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"wandb_enabled": true,
|
| 33 |
+
"wandb_project": "stochastic_sparsity",
|
| 34 |
+
"wandb_name": "365m edge-dense alternating-mixed seed49",
|
| 35 |
+
"wandb_tags": [
|
| 36 |
+
"qwen3",
|
| 37 |
+
"365m",
|
| 38 |
+
"8k",
|
| 39 |
+
"edge-dense",
|
| 40 |
+
"alternating-mixed",
|
| 41 |
+
"seed49"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": null,
|
| 50 |
+
"rope_train_position_scale_set": null,
|
| 51 |
+
"attention_type": "sparse_stochastic",
|
| 52 |
+
"sparse_block_size": 128,
|
| 53 |
+
"sparse_num_random_blocks": 31,
|
| 54 |
+
"sparse_mask_pool_size": 10000,
|
| 55 |
+
"sparse_val_dense_compare": false,
|
| 56 |
+
"sparse_layer_indices": [
|
| 57 |
+
3,
|
| 58 |
+
5,
|
| 59 |
+
7,
|
| 60 |
+
9,
|
| 61 |
+
11,
|
| 62 |
+
13,
|
| 63 |
+
15,
|
| 64 |
+
17,
|
| 65 |
+
19,
|
| 66 |
+
21,
|
| 67 |
+
23
|
| 68 |
+
],
|
| 69 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 70 |
+
"sparse_mask_seed": 0,
|
| 71 |
+
"sparse_mixed_nrb_set": [
|
| 72 |
+
63,
|
| 73 |
+
47,
|
| 74 |
+
31,
|
| 75 |
+
21,
|
| 76 |
+
11
|
| 77 |
+
],
|
| 78 |
+
"dense_mode": false,
|
| 79 |
+
"dense_step_intervals": null
|
| 80 |
+
}
|
| 81 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 82 |
+
Initializing model from scratch (random weights)...
|
| 83 |
+
Checkpoint activations: False
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
================================================================================
|
| 87 |
+
MODEL ARCHITECTURE
|
| 88 |
+
================================================================================
|
| 89 |
+
Model config:
|
| 90 |
+
vocab_size: 151936
|
| 91 |
+
hidden_size: 768
|
| 92 |
+
num_hidden_layers: 28
|
| 93 |
+
num_attention_heads: 6
|
| 94 |
+
num_key_value_heads: 2
|
| 95 |
+
head_dim: 128
|
| 96 |
+
intermediate_size: 3072
|
| 97 |
+
max_position_embeddings: 32768
|
| 98 |
+
rope_theta: 1000000
|
| 99 |
+
rope_train_position_scale_set: None
|
| 100 |
+
attention_type: sparse_stochastic
|
| 101 |
+
sparse_block_size: 128
|
| 102 |
+
sparse_num_random_blocks: 31
|
| 103 |
+
sparse_mask_pool_size: 10000
|
| 104 |
+
sparse_max_seq_length: 8192
|
| 105 |
+
|
| 106 |
+
Parameter count:
|
| 107 |
+
Total parameters: 358,958,848
|
| 108 |
+
Trainable parameters: 358,958,848
|
| 109 |
+
Non-trainable parameters: 0
|
| 110 |
+
Model size (BF16): 0.72 GB
|
| 111 |
+
================================================================================
|
| 112 |
+
|
| 113 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 114 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 115 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 116 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 117 |
+
Dataset split: 3027764 train, 159356 validation
|
| 118 |
+
Training examples (packed, train split): 3027764
|
| 119 |
+
Validation examples (packed, val split): 159356
|
| 120 |
+
Per-device batch size: 2
|
| 121 |
+
World size (GPUs): 1
|
| 122 |
+
Gradient accumulation steps: 10
|
| 123 |
+
Effective batch size (seqs/step): 20
|
| 124 |
+
Sequence length: 8192
|
| 125 |
+
Tokens per optimizer step: 163,840
|
| 126 |
+
Micro-batches per epoch (per rank): 1513882
|
| 127 |
+
Optimizer steps per epoch: 151388
|
| 128 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 129 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 130 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 131 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 132 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_192830-6sx9f1kf
|
| 133 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 134 |
+
wandb: Syncing run 365m edge-dense alternating-mixed seed49
|
| 135 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 136 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/6sx9f1kf
|
| 137 |
+
Starting training...
|
| 138 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 139 |
+
Step 10/44000 | Loss: 12.0711 | LR: 6.67e-06 | 7337.1ms/step | 2.7 samples/s
|
| 140 |
+
Step 20/44000 | Loss: 11.9717 | LR: 1.33e-05 | 1348.8ms/step | 14.8 samples/s
|
| 141 |
+
Step 30/44000 | Loss: 11.6495 | LR: 2.00e-05 | 1349.9ms/step | 14.8 samples/s
|
| 142 |
+
Step 40/44000 | Loss: 11.1050 | LR: 2.67e-05 | 1348.7ms/step | 14.8 samples/s
|
| 143 |
+
Step 50/44000 | Loss: 10.7634 | LR: 3.33e-05 | 1345.1ms/step | 14.9 samples/s
|
| 144 |
+
Step 60/44000 | Loss: 10.5279 | LR: 4.00e-05 | 1349.4ms/step | 14.8 samples/s
|
| 145 |
+
Step 70/44000 | Loss: 10.2709 | LR: 4.67e-05 | 1347.0ms/step | 14.8 samples/s
|
| 146 |
+
Step 80/44000 | Loss: 9.9987 | LR: 5.33e-05 | 1346.2ms/step | 14.9 samples/s
|
| 147 |
+
Step 90/44000 | Loss: 9.6598 | LR: 6.00e-05 | 1351.3ms/step | 14.8 samples/s
|
| 148 |
+
Step 100/44000 | Loss: 9.1523 | LR: 6.67e-05 | 1354.3ms/step | 14.8 samples/s
|
| 149 |
+
Step 110/44000 | Loss: 8.6439 | LR: 7.33e-05 | 1350.7ms/step | 14.8 samples/s
|
| 150 |
+
Step 120/44000 | Loss: 8.1440 | LR: 8.00e-05 | 1349.4ms/step | 14.8 samples/s
|
| 151 |
+
Step 130/44000 | Loss: 7.7750 | LR: 8.67e-05 | 1349.2ms/step | 14.8 samples/s
|
| 152 |
+
Step 140/44000 | Loss: 7.5434 | LR: 9.33e-05 | 1349.3ms/step | 14.8 samples/s
|
| 153 |
+
Step 150/44000 | Loss: 7.3708 | LR: 1.00e-04 | 1351.9ms/step | 14.8 samples/s
|
| 154 |
+
Step 160/44000 | Loss: 7.2375 | LR: 1.07e-04 | 1352.3ms/step | 14.8 samples/s
|
| 155 |
+
Step 170/44000 | Loss: 7.1023 | LR: 1.13e-04 | 1352.8ms/step | 14.8 samples/s
|
| 156 |
+
Step 180/44000 | Loss: 7.0167 | LR: 1.20e-04 | 1358.7ms/step | 14.7 samples/s
|
| 157 |
+
Step 190/44000 | Loss: 6.9254 | LR: 1.27e-04 | 1357.2ms/step | 14.7 samples/s
|
| 158 |
+
Step 200/44000 | Loss: 6.8553 | LR: 1.33e-04 | 1360.8ms/step | 14.7 samples/s
|
| 159 |
+
Step 210/44000 | Loss: 6.7191 | LR: 1.40e-04 | 1361.9ms/step | 14.7 samples/s
|
| 160 |
+
Step 220/44000 | Loss: 6.6928 | LR: 1.47e-04 | 1368.2ms/step | 14.6 samples/s
|
| 161 |
+
Step 230/44000 | Loss: 6.5972 | LR: 1.53e-04 | 1361.7ms/step | 14.7 samples/s
|
| 162 |
+
Step 240/44000 | Loss: 6.5085 | LR: 1.60e-04 | 1359.7ms/step | 14.7 samples/s
|
| 163 |
+
Step 250/44000 | Loss: 6.4586 | LR: 1.67e-04 | 1366.0ms/step | 14.6 samples/s
|
| 164 |
+
Validation at step 250: Loss = 6.4323, Perplexity = 621.61
|
| 165 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-250
|
| 166 |
+
Step 260/44000 | Loss: 6.4339 | LR: 1.73e-04 | 6337.9ms/step | 3.2 samples/s
|
| 167 |
+
Step 270/44000 | Loss: 6.3622 | LR: 1.80e-04 | 1361.0ms/step | 14.7 samples/s
|
| 168 |
+
Step 280/44000 | Loss: 6.3080 | LR: 1.87e-04 | 1368.4ms/step | 14.6 samples/s
|
| 169 |
+
Step 290/44000 | Loss: 6.2640 | LR: 1.93e-04 | 1363.1ms/step | 14.7 samples/s
|
| 170 |
+
Step 300/44000 | Loss: 6.2404 | LR: 2.00e-04 | 1361.7ms/step | 14.7 samples/s
|
| 171 |
+
Step 310/44000 | Loss: 6.1606 | LR: 2.07e-04 | 1362.2ms/step | 14.7 samples/s
|
| 172 |
+
Step 320/44000 | Loss: 6.1321 | LR: 2.13e-04 | 1357.4ms/step | 14.7 samples/s
|
| 173 |
+
Step 330/44000 | Loss: 6.0755 | LR: 2.20e-04 | 1366.1ms/step | 14.6 samples/s
|
| 174 |
+
Step 340/44000 | Loss: 6.0630 | LR: 2.27e-04 | 1360.5ms/step | 14.7 samples/s
|
| 175 |
+
Step 350/44000 | Loss: 6.0028 | LR: 2.33e-04 | 1362.8ms/step | 14.7 samples/s
|
| 176 |
+
Step 360/44000 | Loss: 5.9657 | LR: 2.40e-04 | 1368.0ms/step | 14.6 samples/s
|
| 177 |
+
Step 370/44000 | Loss: 5.9471 | LR: 2.47e-04 | 1363.4ms/step | 14.7 samples/s
|
| 178 |
+
Step 380/44000 | Loss: 5.9143 | LR: 2.53e-04 | 1364.3ms/step | 14.7 samples/s
|
| 179 |
+
Step 390/44000 | Loss: 5.8743 | LR: 2.60e-04 | 1363.4ms/step | 14.7 samples/s
|
| 180 |
+
Step 400/44000 | Loss: 5.8099 | LR: 2.67e-04 | 1360.2ms/step | 14.7 samples/s
|
| 181 |
+
Step 410/44000 | Loss: 5.7998 | LR: 2.73e-04 | 1364.9ms/step | 14.7 samples/s
|
| 182 |
+
Step 420/44000 | Loss: 5.7829 | LR: 2.80e-04 | 1365.5ms/step | 14.6 samples/s
|
| 183 |
+
Step 430/44000 | Loss: 5.7467 | LR: 2.87e-04 | 1368.5ms/step | 14.6 samples/s
|
| 184 |
+
Step 440/44000 | Loss: 5.7170 | LR: 2.93e-04 | 1364.4ms/step | 14.7 samples/s
|
| 185 |
+
Step 450/44000 | Loss: 5.6652 | LR: 3.00e-04 | 1361.5ms/step | 14.7 samples/s
|
| 186 |
+
Step 460/44000 | Loss: 5.6324 | LR: 3.07e-04 | 1361.8ms/step | 14.7 samples/s
|
| 187 |
+
Step 470/44000 | Loss: 5.6002 | LR: 3.13e-04 | 1356.9ms/step | 14.7 samples/s
|
| 188 |
+
Step 480/44000 | Loss: 5.5505 | LR: 3.20e-04 | 1357.6ms/step | 14.7 samples/s
|
| 189 |
+
Step 490/44000 | Loss: 5.5154 | LR: 3.27e-04 | 1357.4ms/step | 14.7 samples/s
|
| 190 |
+
Step 500/44000 | Loss: 5.5163 | LR: 3.33e-04 | 1357.4ms/step | 14.7 samples/s
|
| 191 |
+
Validation at step 500: Loss = 5.4815, Perplexity = 240.20
|
| 192 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-500
|
| 193 |
+
Step 510/44000 | Loss: 5.4842 | LR: 3.40e-04 | 3893.2ms/step | 5.1 samples/s
|
| 194 |
+
Step 520/44000 | Loss: 5.4256 | LR: 3.47e-04 | 1367.2ms/step | 14.6 samples/s
|
| 195 |
+
Step 530/44000 | Loss: 5.4154 | LR: 3.53e-04 | 1368.3ms/step | 14.6 samples/s
|
| 196 |
+
Step 540/44000 | Loss: 5.3837 | LR: 3.60e-04 | 1362.0ms/step | 14.7 samples/s
|
| 197 |
+
Step 550/44000 | Loss: 5.3384 | LR: 3.67e-04 | 1363.4ms/step | 14.7 samples/s
|
| 198 |
+
Step 560/44000 | Loss: 5.3207 | LR: 3.73e-04 | 1365.5ms/step | 14.6 samples/s
|
| 199 |
+
Step 570/44000 | Loss: 5.2742 | LR: 3.80e-04 | 1367.1ms/step | 14.6 samples/s
|
| 200 |
+
Step 580/44000 | Loss: 5.2714 | LR: 3.87e-04 | 1362.7ms/step | 14.7 samples/s
|
| 201 |
+
Step 590/44000 | Loss: 5.2158 | LR: 3.93e-04 | 1362.5ms/step | 14.7 samples/s
|
| 202 |
+
Step 600/44000 | Loss: 5.2123 | LR: 4.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 203 |
+
Step 610/44000 | Loss: 5.1481 | LR: 4.07e-04 | 1363.8ms/step | 14.7 samples/s
|
| 204 |
+
Step 620/44000 | Loss: 5.1340 | LR: 4.13e-04 | 1364.8ms/step | 14.7 samples/s
|
| 205 |
+
Step 630/44000 | Loss: 5.1076 | LR: 4.20e-04 | 1361.0ms/step | 14.7 samples/s
|
| 206 |
+
Step 640/44000 | Loss: 5.1054 | LR: 4.27e-04 | 1359.0ms/step | 14.7 samples/s
|
| 207 |
+
Step 650/44000 | Loss: 5.0491 | LR: 4.33e-04 | 1364.2ms/step | 14.7 samples/s
|
| 208 |
+
Step 660/44000 | Loss: 5.0562 | LR: 4.40e-04 | 1362.1ms/step | 14.7 samples/s
|
| 209 |
+
Step 670/44000 | Loss: 5.0276 | LR: 4.47e-04 | 1362.4ms/step | 14.7 samples/s
|
| 210 |
+
Step 680/44000 | Loss: 4.9559 | LR: 4.53e-04 | 1363.4ms/step | 14.7 samples/s
|
| 211 |
+
Step 690/44000 | Loss: 4.9932 | LR: 4.60e-04 | 1365.8ms/step | 14.6 samples/s
|
| 212 |
+
Step 700/44000 | Loss: 4.9010 | LR: 4.67e-04 | 1366.0ms/step | 14.6 samples/s
|
| 213 |
+
Step 710/44000 | Loss: 4.9001 | LR: 4.73e-04 | 1364.0ms/step | 14.7 samples/s
|
| 214 |
+
Step 720/44000 | Loss: 4.8877 | LR: 4.80e-04 | 1358.9ms/step | 14.7 samples/s
|
| 215 |
+
Step 730/44000 | Loss: 4.8307 | LR: 4.87e-04 | 1360.0ms/step | 14.7 samples/s
|
| 216 |
+
Step 740/44000 | Loss: 4.8304 | LR: 4.93e-04 | 1360.6ms/step | 14.7 samples/s
|
| 217 |
+
Step 750/44000 | Loss: 4.7977 | LR: 5.00e-04 | 1364.2ms/step | 14.7 samples/s
|
| 218 |
+
Validation at step 750: Loss = 4.7705, Perplexity = 117.98
|
| 219 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-750
|
| 220 |
+
Step 760/44000 | Loss: 4.7924 | LR: 5.00e-04 | 3881.2ms/step | 5.2 samples/s
|
| 221 |
+
Step 770/44000 | Loss: 4.7358 | LR: 5.00e-04 | 1360.7ms/step | 14.7 samples/s
|
| 222 |
+
Step 780/44000 | Loss: 4.6945 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 223 |
+
Step 790/44000 | Loss: 4.6635 | LR: 5.00e-04 | 1358.9ms/step | 14.7 samples/s
|
| 224 |
+
Step 800/44000 | Loss: 4.6523 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 225 |
+
Step 810/44000 | Loss: 4.6363 | LR: 5.00e-04 | 1364.8ms/step | 14.7 samples/s
|
| 226 |
+
Step 820/44000 | Loss: 4.6026 | LR: 5.00e-04 | 1364.5ms/step | 14.7 samples/s
|
| 227 |
+
Step 830/44000 | Loss: 4.5735 | LR: 5.00e-04 | 1363.0ms/step | 14.7 samples/s
|
| 228 |
+
Step 840/44000 | Loss: 4.5417 | LR: 5.00e-04 | 1369.5ms/step | 14.6 samples/s
|
| 229 |
+
Step 850/44000 | Loss: 4.5144 | LR: 5.00e-04 | 1363.5ms/step | 14.7 samples/s
|
| 230 |
+
Step 860/44000 | Loss: 4.4845 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 231 |
+
Step 870/44000 | Loss: 4.4991 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 232 |
+
Step 880/44000 | Loss: 4.4861 | LR: 5.00e-04 | 1364.0ms/step | 14.7 samples/s
|
| 233 |
+
Step 890/44000 | Loss: 4.4595 | LR: 5.00e-04 | 1362.2ms/step | 14.7 samples/s
|
| 234 |
+
Step 900/44000 | Loss: 4.4329 | LR: 5.00e-04 | 1360.2ms/step | 14.7 samples/s
|
| 235 |
+
Step 910/44000 | Loss: 4.4364 | LR: 5.00e-04 | 1361.7ms/step | 14.7 samples/s
|
| 236 |
+
Step 920/44000 | Loss: 4.3790 | LR: 5.00e-04 | 1365.4ms/step | 14.6 samples/s
|
| 237 |
+
Step 930/44000 | Loss: 4.3906 | LR: 5.00e-04 | 1364.6ms/step | 14.7 samples/s
|
| 238 |
+
Step 940/44000 | Loss: 4.3651 | LR: 5.00e-04 | 1364.3ms/step | 14.7 samples/s
|
| 239 |
+
Step 950/44000 | Loss: 4.3562 | LR: 5.00e-04 | 1367.6ms/step | 14.6 samples/s
|
| 240 |
+
Step 960/44000 | Loss: 4.3581 | LR: 5.00e-04 | 1362.6ms/step | 14.7 samples/s
|
| 241 |
+
Step 970/44000 | Loss: 4.3182 | LR: 5.00e-04 | 1362.7ms/step | 14.7 samples/s
|
| 242 |
+
Step 980/44000 | Loss: 4.3303 | LR: 5.00e-04 | 1364.4ms/step | 14.7 samples/s
|
| 243 |
+
Step 990/44000 | Loss: 4.2891 | LR: 5.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 244 |
+
Step 1000/44000 | Loss: 4.2516 | LR: 5.00e-04 | 1364.5ms/step | 14.7 samples/s
|
| 245 |
+
Validation at step 1000: Loss = 4.2733, Perplexity = 71.76
|
| 246 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1000
|
| 247 |
+
Step 1010/44000 | Loss: 4.2714 | LR: 5.00e-04 | 3858.3ms/step | 5.2 samples/s
|
| 248 |
+
Step 1020/44000 | Loss: 4.2798 | LR: 5.00e-04 | 1362.7ms/step | 14.7 samples/s
|
| 249 |
+
Step 1030/44000 | Loss: 4.2469 | LR: 5.00e-04 | 1359.1ms/step | 14.7 samples/s
|
| 250 |
+
Step 1040/44000 | Loss: 4.2219 | LR: 5.00e-04 | 1367.3ms/step | 14.6 samples/s
|
| 251 |
+
Step 1050/44000 | Loss: 4.2022 | LR: 5.00e-04 | 1366.8ms/step | 14.6 samples/s
|
| 252 |
+
Step 1060/44000 | Loss: 4.2040 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 253 |
+
Step 1070/44000 | Loss: 4.1619 | LR: 5.00e-04 | 1364.8ms/step | 14.7 samples/s
|
| 254 |
+
Step 1080/44000 | Loss: 4.1578 | LR: 5.00e-04 | 1365.3ms/step | 14.6 samples/s
|
| 255 |
+
Step 1090/44000 | Loss: 4.2069 | LR: 5.00e-04 | 1362.6ms/step | 14.7 samples/s
|
| 256 |
+
Step 1100/44000 | Loss: 4.1523 | LR: 5.00e-04 | 1361.9ms/step | 14.7 samples/s
|
| 257 |
+
Step 1110/44000 | Loss: 4.1572 | LR: 5.00e-04 | 1361.0ms/step | 14.7 samples/s
|
| 258 |
+
Step 1120/44000 | Loss: 4.1286 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 259 |
+
Step 1130/44000 | Loss: 4.1544 | LR: 5.00e-04 | 1364.0ms/step | 14.7 samples/s
|
| 260 |
+
Step 1140/44000 | Loss: 4.0852 | LR: 5.00e-04 | 1365.0ms/step | 14.7 samples/s
|
| 261 |
+
Step 1150/44000 | Loss: 4.1095 | LR: 5.00e-04 | 1368.8ms/step | 14.6 samples/s
|
| 262 |
+
Step 1160/44000 | Loss: 4.0869 | LR: 5.00e-04 | 1365.0ms/step | 14.7 samples/s
|
| 263 |
+
Step 1170/44000 | Loss: 4.1058 | LR: 5.00e-04 | 1361.2ms/step | 14.7 samples/s
|
| 264 |
+
Step 1180/44000 | Loss: 4.0906 | LR: 5.00e-04 | 1365.3ms/step | 14.6 samples/s
|
| 265 |
+
Step 1190/44000 | Loss: 4.0867 | LR: 5.00e-04 | 1359.5ms/step | 14.7 samples/s
|
| 266 |
+
Step 1200/44000 | Loss: 4.0732 | LR: 5.00e-04 | 1359.0ms/step | 14.7 samples/s
|
| 267 |
+
Step 1210/44000 | Loss: 4.0530 | LR: 5.00e-04 | 1356.4ms/step | 14.7 samples/s
|
| 268 |
+
Step 1220/44000 | Loss: 4.0610 | LR: 5.00e-04 | 1357.3ms/step | 14.7 samples/s
|
| 269 |
+
Step 1230/44000 | Loss: 4.0312 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 270 |
+
Step 1240/44000 | Loss: 4.0391 | LR: 5.00e-04 | 1357.0ms/step | 14.7 samples/s
|
| 271 |
+
Step 1250/44000 | Loss: 3.9951 | LR: 5.00e-04 | 1356.5ms/step | 14.7 samples/s
|
| 272 |
+
Validation at step 1250: Loss = 4.0120, Perplexity = 55.26
|
| 273 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1250
|
| 274 |
+
Step 1260/44000 | Loss: 4.0145 | LR: 5.00e-04 | 3857.9ms/step | 5.2 samples/s
|
| 275 |
+
Step 1270/44000 | Loss: 4.0223 | LR: 5.00e-04 | 1354.0ms/step | 14.8 samples/s
|
| 276 |
+
Step 1280/44000 | Loss: 4.0026 | LR: 5.00e-04 | 1355.7ms/step | 14.8 samples/s
|
| 277 |
+
Step 1290/44000 | Loss: 3.9988 | LR: 5.00e-04 | 1355.0ms/step | 14.8 samples/s
|
| 278 |
+
Step 1300/44000 | Loss: 3.9714 | LR: 5.00e-04 | 1353.9ms/step | 14.8 samples/s
|
| 279 |
+
Step 1310/44000 | Loss: 3.9568 | LR: 5.00e-04 | 1353.0ms/step | 14.8 samples/s
|
| 280 |
+
Step 1320/44000 | Loss: 3.9634 | LR: 5.00e-04 | 1353.1ms/step | 14.8 samples/s
|
| 281 |
+
Step 1330/44000 | Loss: 3.9521 | LR: 5.00e-04 | 1354.6ms/step | 14.8 samples/s
|
| 282 |
+
Step 1340/44000 | Loss: 3.9543 | LR: 5.00e-04 | 1356.6ms/step | 14.7 samples/s
|
| 283 |
+
Step 1350/44000 | Loss: 3.9717 | LR: 5.00e-04 | 1352.0ms/step | 14.8 samples/s
|
| 284 |
+
Step 1360/44000 | Loss: 3.9713 | LR: 5.00e-04 | 1352.8ms/step | 14.8 samples/s
|
| 285 |
+
Step 1370/44000 | Loss: 3.9478 | LR: 5.00e-04 | 1356.2ms/step | 14.7 samples/s
|
| 286 |
+
Step 1380/44000 | Loss: 3.9247 | LR: 5.00e-04 | 1353.5ms/step | 14.8 samples/s
|
| 287 |
+
Step 1390/44000 | Loss: 3.9088 | LR: 5.00e-04 | 1355.6ms/step | 14.8 samples/s
|
| 288 |
+
Step 1400/44000 | Loss: 3.9230 | LR: 5.00e-04 | 1355.7ms/step | 14.8 samples/s
|
| 289 |
+
Step 1410/44000 | Loss: 3.9060 | LR: 5.00e-04 | 1353.5ms/step | 14.8 samples/s
|
| 290 |
+
Step 1420/44000 | Loss: 3.9135 | LR: 5.00e-04 | 1353.1ms/step | 14.8 samples/s
|
| 291 |
+
Step 1430/44000 | Loss: 3.8995 | LR: 5.00e-04 | 1350.9ms/step | 14.8 samples/s
|
| 292 |
+
Step 1440/44000 | Loss: 3.8967 | LR: 5.00e-04 | 1352.4ms/step | 14.8 samples/s
|
| 293 |
+
Step 1450/44000 | Loss: 3.8772 | LR: 5.00e-04 | 1351.6ms/step | 14.8 samples/s
|
| 294 |
+
Step 1460/44000 | Loss: 3.8708 | LR: 5.00e-04 | 1351.6ms/step | 14.8 samples/s
|
| 295 |
+
Step 1470/44000 | Loss: 3.8727 | LR: 5.00e-04 | 1351.7ms/step | 14.8 samples/s
|
| 296 |
+
Step 1480/44000 | Loss: 3.8852 | LR: 5.00e-04 | 1350.0ms/step | 14.8 samples/s
|
| 297 |
+
Step 1490/44000 | Loss: 3.8835 | LR: 5.00e-04 | 1350.0ms/step | 14.8 samples/s
|
| 298 |
+
Step 1500/44000 | Loss: 3.8918 | LR: 5.00e-04 | 1353.4ms/step | 14.8 samples/s
|
| 299 |
+
Validation at step 1500: Loss = 3.8619, Perplexity = 47.55
|
| 300 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1500
|
| 301 |
+
Step 1510/44000 | Loss: 3.8767 | LR: 5.00e-04 | 3844.1ms/step | 5.2 samples/s
|
| 302 |
+
Step 1520/44000 | Loss: 3.8616 | LR: 5.00e-04 | 1353.3ms/step | 14.8 samples/s
|
| 303 |
+
Step 1530/44000 | Loss: 3.8317 | LR: 5.00e-04 | 1349.9ms/step | 14.8 samples/s
|
| 304 |
+
Step 1540/44000 | Loss: 3.8673 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 305 |
+
Step 1550/44000 | Loss: 3.8586 | LR: 5.00e-04 | 1360.5ms/step | 14.7 samples/s
|
| 306 |
+
Step 1560/44000 | Loss: 3.8401 | LR: 5.00e-04 | 1355.9ms/step | 14.7 samples/s
|
| 307 |
+
Step 1570/44000 | Loss: 3.8208 | LR: 5.00e-04 | 1357.2ms/step | 14.7 samples/s
|
| 308 |
+
Step 1580/44000 | Loss: 3.8636 | LR: 5.00e-04 | 1352.6ms/step | 14.8 samples/s
|
| 309 |
+
Step 1590/44000 | Loss: 3.8059 | LR: 5.00e-04 | 1354.6ms/step | 14.8 samples/s
|
| 310 |
+
Step 1600/44000 | Loss: 3.8345 | LR: 5.00e-04 | 1353.4ms/step | 14.8 samples/s
|
| 311 |
+
Step 1610/44000 | Loss: 3.8351 | LR: 5.00e-04 | 1352.3ms/step | 14.8 samples/s
|
| 312 |
+
Step 1620/44000 | Loss: 3.7816 | LR: 5.00e-04 | 1354.8ms/step | 14.8 samples/s
|
| 313 |
+
Step 1630/44000 | Loss: 3.7856 | LR: 4.99e-04 | 1354.5ms/step | 14.8 samples/s
|
| 314 |
+
Step 1640/44000 | Loss: 3.7963 | LR: 4.99e-04 | 1352.8ms/step | 14.8 samples/s
|
| 315 |
+
Step 1650/44000 | Loss: 3.7829 | LR: 4.99e-04 | 1351.4ms/step | 14.8 samples/s
|
| 316 |
+
Step 1660/44000 | Loss: 3.7934 | LR: 4.99e-04 | 1357.3ms/step | 14.7 samples/s
|
| 317 |
+
Step 1670/44000 | Loss: 3.8035 | LR: 4.99e-04 | 1351.3ms/step | 14.8 samples/s
|
| 318 |
+
Step 1680/44000 | Loss: 3.7785 | LR: 4.99e-04 | 1354.9ms/step | 14.8 samples/s
|
| 319 |
+
Step 1690/44000 | Loss: 3.7695 | LR: 4.99e-04 | 1351.1ms/step | 14.8 samples/s
|
| 320 |
+
Step 1700/44000 | Loss: 3.7499 | LR: 4.99e-04 | 1352.3ms/step | 14.8 samples/s
|
| 321 |
+
Step 1710/44000 | Loss: 3.7566 | LR: 4.99e-04 | 1350.2ms/step | 14.8 samples/s
|
| 322 |
+
Step 1720/44000 | Loss: 3.7609 | LR: 4.99e-04 | 1350.5ms/step | 14.8 samples/s
|
| 323 |
+
Step 1730/44000 | Loss: 3.7503 | LR: 4.99e-04 | 1358.4ms/step | 14.7 samples/s
|
| 324 |
+
Step 1740/44000 | Loss: 3.7779 | LR: 4.99e-04 | 1352.1ms/step | 14.8 samples/s
|
| 325 |
+
Step 1750/44000 | Loss: 3.7930 | LR: 4.99e-04 | 1355.4ms/step | 14.8 samples/s
|
| 326 |
+
Validation at step 1750: Loss = 3.7526, Perplexity = 42.63
|
| 327 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-1750
|
| 328 |
+
Step 1760/44000 | Loss: 3.7621 | LR: 4.99e-04 | 3886.9ms/step | 5.1 samples/s
|
| 329 |
+
Step 1770/44000 | Loss: 3.7366 | LR: 4.99e-04 | 1358.5ms/step | 14.7 samples/s
|
| 330 |
+
Step 1780/44000 | Loss: 3.7455 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 331 |
+
Step 1790/44000 | Loss: 3.7231 | LR: 4.99e-04 | 1365.1ms/step | 14.7 samples/s
|
| 332 |
+
Step 1800/44000 | Loss: 3.7170 | LR: 4.99e-04 | 1363.6ms/step | 14.7 samples/s
|
| 333 |
+
Step 1810/44000 | Loss: 3.7430 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 334 |
+
Step 1820/44000 | Loss: 3.7280 | LR: 4.99e-04 | 1359.1ms/step | 14.7 samples/s
|
| 335 |
+
Step 1830/44000 | Loss: 3.7129 | LR: 4.99e-04 | 1356.7ms/step | 14.7 samples/s
|
| 336 |
+
Step 1840/44000 | Loss: 3.7172 | LR: 4.99e-04 | 1360.2ms/step | 14.7 samples/s
|
| 337 |
+
Step 1850/44000 | Loss: 3.7115 | LR: 4.99e-04 | 1358.9ms/step | 14.7 samples/s
|
| 338 |
+
Step 1860/44000 | Loss: 3.7145 | LR: 4.99e-04 | 1357.2ms/step | 14.7 samples/s
|
| 339 |
+
Step 1870/44000 | Loss: 3.7432 | LR: 4.99e-04 | 1361.3ms/step | 14.7 samples/s
|
| 340 |
+
Step 1880/44000 | Loss: 3.7112 | LR: 4.99e-04 | 1359.7ms/step | 14.7 samples/s
|
| 341 |
+
Step 1890/44000 | Loss: 3.6872 | LR: 4.99e-04 | 1360.2ms/step | 14.7 samples/s
|
| 342 |
+
Step 1900/44000 | Loss: 3.7188 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 343 |
+
Step 1910/44000 | Loss: 3.6731 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 344 |
+
Step 1920/44000 | Loss: 3.6885 | LR: 4.99e-04 | 1358.6ms/step | 14.7 samples/s
|
| 345 |
+
Step 1930/44000 | Loss: 3.6980 | LR: 4.99e-04 | 1357.0ms/step | 14.7 samples/s
|
| 346 |
+
Step 1940/44000 | Loss: 3.6770 | LR: 4.99e-04 | 1356.6ms/step | 14.7 samples/s
|
| 347 |
+
Step 1950/44000 | Loss: 3.6759 | LR: 4.99e-04 | 1358.3ms/step | 14.7 samples/s
|
| 348 |
+
Step 1960/44000 | Loss: 3.6877 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 349 |
+
Step 1970/44000 | Loss: 3.6887 | LR: 4.99e-04 | 1362.6ms/step | 14.7 samples/s
|
| 350 |
+
Step 1980/44000 | Loss: 3.6511 | LR: 4.99e-04 | 1360.4ms/step | 14.7 samples/s
|
| 351 |
+
Step 1990/44000 | Loss: 3.6831 | LR: 4.99e-04 | 1359.3ms/step | 14.7 samples/s
|
| 352 |
+
Step 2000/44000 | Loss: 3.6669 | LR: 4.99e-04 | 1356.7ms/step | 14.7 samples/s
|
| 353 |
+
Validation at step 2000: Loss = 3.6727, Perplexity = 39.36
|
| 354 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2000
|
| 355 |
+
Step 2010/44000 | Loss: 3.6578 | LR: 4.99e-04 | 3858.1ms/step | 5.2 samples/s
|
| 356 |
+
Step 2020/44000 | Loss: 3.6638 | LR: 4.99e-04 | 1359.7ms/step | 14.7 samples/s
|
| 357 |
+
Step 2030/44000 | Loss: 3.6607 | LR: 4.99e-04 | 1360.6ms/step | 14.7 samples/s
|
| 358 |
+
Step 2040/44000 | Loss: 3.6861 | LR: 4.99e-04 | 1366.5ms/step | 14.6 samples/s
|
| 359 |
+
Step 2050/44000 | Loss: 3.6554 | LR: 4.99e-04 | 1366.1ms/step | 14.6 samples/s
|
| 360 |
+
Step 2060/44000 | Loss: 3.6552 | LR: 4.99e-04 | 1357.6ms/step | 14.7 samples/s
|
| 361 |
+
Step 2070/44000 | Loss: 3.6447 | LR: 4.99e-04 | 1361.8ms/step | 14.7 samples/s
|
| 362 |
+
Step 2080/44000 | Loss: 3.6344 | LR: 4.99e-04 | 1362.7ms/step | 14.7 samples/s
|
| 363 |
+
Step 2090/44000 | Loss: 3.6519 | LR: 4.99e-04 | 1364.7ms/step | 14.7 samples/s
|
| 364 |
+
Step 2100/44000 | Loss: 3.6354 | LR: 4.99e-04 | 1363.7ms/step | 14.7 samples/s
|
| 365 |
+
Step 2110/44000 | Loss: 3.6871 | LR: 4.99e-04 | 1363.4ms/step | 14.7 samples/s
|
| 366 |
+
Step 2120/44000 | Loss: 3.6250 | LR: 4.99e-04 | 1365.0ms/step | 14.7 samples/s
|
| 367 |
+
Step 2130/44000 | Loss: 3.6637 | LR: 4.99e-04 | 1362.6ms/step | 14.7 samples/s
|
| 368 |
+
Step 2140/44000 | Loss: 3.6473 | LR: 4.99e-04 | 1363.6ms/step | 14.7 samples/s
|
| 369 |
+
Step 2150/44000 | Loss: 3.6361 | LR: 4.99e-04 | 1358.8ms/step | 14.7 samples/s
|
| 370 |
+
Step 2160/44000 | Loss: 3.6796 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 371 |
+
Step 2170/44000 | Loss: 3.6585 | LR: 4.99e-04 | 1361.2ms/step | 14.7 samples/s
|
| 372 |
+
Step 2180/44000 | Loss: 3.6202 | LR: 4.99e-04 | 1364.7ms/step | 14.7 samples/s
|
| 373 |
+
Step 2190/44000 | Loss: 3.5909 | LR: 4.99e-04 | 1362.8ms/step | 14.7 samples/s
|
| 374 |
+
Step 2200/44000 | Loss: 3.6456 | LR: 4.99e-04 | 1364.9ms/step | 14.7 samples/s
|
| 375 |
+
Step 2210/44000 | Loss: 3.6462 | LR: 4.99e-04 | 1362.4ms/step | 14.7 samples/s
|
| 376 |
+
Step 2220/44000 | Loss: 3.6216 | LR: 4.99e-04 | 1360.7ms/step | 14.7 samples/s
|
| 377 |
+
Step 2230/44000 | Loss: 3.5979 | LR: 4.99e-04 | 1355.0ms/step | 14.8 samples/s
|
| 378 |
+
Step 2240/44000 | Loss: 3.5852 | LR: 4.99e-04 | 1358.8ms/step | 14.7 samples/s
|
| 379 |
+
Step 2250/44000 | Loss: 3.6189 | LR: 4.99e-04 | 1356.9ms/step | 14.7 samples/s
|
| 380 |
+
Validation at step 2250: Loss = 3.6066, Perplexity = 36.84
|
| 381 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2250
|
| 382 |
+
Step 2260/44000 | Loss: 3.6230 | LR: 4.98e-04 | 3865.2ms/step | 5.2 samples/s
|
| 383 |
+
Step 2270/44000 | Loss: 3.6361 | LR: 4.98e-04 | 1362.6ms/step | 14.7 samples/s
|
| 384 |
+
Step 2280/44000 | Loss: 3.6061 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 385 |
+
Step 2290/44000 | Loss: 3.6187 | LR: 4.98e-04 | 1359.5ms/step | 14.7 samples/s
|
| 386 |
+
Step 2300/44000 | Loss: 3.6018 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 387 |
+
Step 2310/44000 | Loss: 3.5893 | LR: 4.98e-04 | 1360.6ms/step | 14.7 samples/s
|
| 388 |
+
Step 2320/44000 | Loss: 3.5825 | LR: 4.98e-04 | 1361.1ms/step | 14.7 samples/s
|
| 389 |
+
Step 2330/44000 | Loss: 3.5693 | LR: 4.98e-04 | 1362.1ms/step | 14.7 samples/s
|
| 390 |
+
Step 2340/44000 | Loss: 3.6163 | LR: 4.98e-04 | 1359.4ms/step | 14.7 samples/s
|
| 391 |
+
Step 2350/44000 | Loss: 3.6004 | LR: 4.98e-04 | 1360.6ms/step | 14.7 samples/s
|
| 392 |
+
Step 2360/44000 | Loss: 3.5961 | LR: 4.98e-04 | 1362.2ms/step | 14.7 samples/s
|
| 393 |
+
Step 2370/44000 | Loss: 3.5617 | LR: 4.98e-04 | 1362.2ms/step | 14.7 samples/s
|
| 394 |
+
Step 2380/44000 | Loss: 3.5395 | LR: 4.98e-04 | 1359.6ms/step | 14.7 samples/s
|
| 395 |
+
Step 2390/44000 | Loss: 3.5808 | LR: 4.98e-04 | 1362.8ms/step | 14.7 samples/s
|
| 396 |
+
Step 2400/44000 | Loss: 3.5630 | LR: 4.98e-04 | 1363.6ms/step | 14.7 samples/s
|
| 397 |
+
Step 2410/44000 | Loss: 3.5863 | LR: 4.98e-04 | 1365.5ms/step | 14.6 samples/s
|
| 398 |
+
Step 2420/44000 | Loss: 3.5684 | LR: 4.98e-04 | 1364.4ms/step | 14.7 samples/s
|
| 399 |
+
Step 2430/44000 | Loss: 3.5871 | LR: 4.98e-04 | 1365.0ms/step | 14.7 samples/s
|
| 400 |
+
Step 2440/44000 | Loss: 3.5826 | LR: 4.98e-04 | 1363.2ms/step | 14.7 samples/s
|
| 401 |
+
Step 2450/44000 | Loss: 3.6135 | LR: 4.98e-04 | 1359.5ms/step | 14.7 samples/s
|
| 402 |
+
Step 2460/44000 | Loss: 3.5397 | LR: 4.98e-04 | 1362.1ms/step | 14.7 samples/s
|
| 403 |
+
Step 2470/44000 | Loss: 3.5845 | LR: 4.98e-04 | 1361.4ms/step | 14.7 samples/s
|
| 404 |
+
Step 2480/44000 | Loss: 3.5921 | LR: 4.98e-04 | 1363.6ms/step | 14.7 samples/s
|
| 405 |
+
Step 2490/44000 | Loss: 3.5702 | LR: 4.98e-04 | 1364.9ms/step | 14.7 samples/s
|
| 406 |
+
Step 2500/44000 | Loss: 3.5535 | LR: 4.98e-04 | 1362.9ms/step | 14.7 samples/s
|
| 407 |
+
Validation at step 2500: Loss = 3.5555, Perplexity = 35.00
|
| 408 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2500
|
| 409 |
+
Step 2510/44000 | Loss: 3.5582 | LR: 4.98e-04 | 3873.3ms/step | 5.2 samples/s
|
| 410 |
+
Step 2520/44000 | Loss: 3.5496 | LR: 4.98e-04 | 1356.9ms/step | 14.7 samples/s
|
| 411 |
+
Step 2530/44000 | Loss: 3.5313 | LR: 4.98e-04 | 1356.0ms/step | 14.7 samples/s
|
| 412 |
+
Step 2540/44000 | Loss: 3.5369 | LR: 4.98e-04 | 1358.0ms/step | 14.7 samples/s
|
| 413 |
+
Step 2550/44000 | Loss: 3.5082 | LR: 4.98e-04 | 1360.9ms/step | 14.7 samples/s
|
| 414 |
+
Step 2560/44000 | Loss: 3.5580 | LR: 4.98e-04 | 1364.1ms/step | 14.7 samples/s
|
| 415 |
+
Step 2570/44000 | Loss: 3.5531 | LR: 4.98e-04 | 1363.7ms/step | 14.7 samples/s
|
| 416 |
+
Step 2580/44000 | Loss: 3.5283 | LR: 4.98e-04 | 1364.4ms/step | 14.7 samples/s
|
| 417 |
+
Step 2590/44000 | Loss: 3.5522 | LR: 4.98e-04 | 1362.0ms/step | 14.7 samples/s
|
| 418 |
+
Step 2600/44000 | Loss: 3.5163 | LR: 4.98e-04 | 1360.8ms/step | 14.7 samples/s
|
| 419 |
+
Step 2610/44000 | Loss: 3.5404 | LR: 4.98e-04 | 1359.1ms/step | 14.7 samples/s
|
| 420 |
+
Step 2620/44000 | Loss: 3.5333 | LR: 4.98e-04 | 1359.1ms/step | 14.7 samples/s
|
| 421 |
+
Step 2630/44000 | Loss: 3.5329 | LR: 4.98e-04 | 1367.4ms/step | 14.6 samples/s
|
| 422 |
+
Step 2640/44000 | Loss: 3.5292 | LR: 4.98e-04 | 1357.1ms/step | 14.7 samples/s
|
| 423 |
+
Step 2650/44000 | Loss: 3.5057 | LR: 4.98e-04 | 1361.4ms/step | 14.7 samples/s
|
| 424 |
+
Step 2660/44000 | Loss: 3.5152 | LR: 4.98e-04 | 1361.8ms/step | 14.7 samples/s
|
| 425 |
+
Step 2670/44000 | Loss: 3.5575 | LR: 4.98e-04 | 1363.0ms/step | 14.7 samples/s
|
| 426 |
+
Step 2680/44000 | Loss: 3.5442 | LR: 4.98e-04 | 1358.0ms/step | 14.7 samples/s
|
| 427 |
+
Step 2690/44000 | Loss: 3.5181 | LR: 4.98e-04 | 1360.0ms/step | 14.7 samples/s
|
| 428 |
+
Step 2700/44000 | Loss: 3.5460 | LR: 4.97e-04 | 1360.9ms/step | 14.7 samples/s
|
| 429 |
+
Step 2710/44000 | Loss: 3.5127 | LR: 4.97e-04 | 1359.6ms/step | 14.7 samples/s
|
| 430 |
+
Step 2720/44000 | Loss: 3.5330 | LR: 4.97e-04 | 1360.0ms/step | 14.7 samples/s
|
| 431 |
+
Step 2730/44000 | Loss: 3.4644 | LR: 4.97e-04 | 1357.0ms/step | 14.7 samples/s
|
| 432 |
+
Step 2740/44000 | Loss: 3.5057 | LR: 4.97e-04 | 1353.9ms/step | 14.8 samples/s
|
| 433 |
+
Step 2750/44000 | Loss: 3.5142 | LR: 4.97e-04 | 1361.5ms/step | 14.7 samples/s
|
| 434 |
+
Validation at step 2750: Loss = 3.5116, Perplexity = 33.50
|
| 435 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750
|
| 436 |
+
Step 2760/44000 | Loss: 3.5153 | LR: 4.97e-04 | 3857.2ms/step | 5.2 samples/s
|
| 437 |
+
Step 2770/44000 | Loss: 3.5200 | LR: 4.97e-04 | 1355.1ms/step | 14.8 samples/s
|
| 438 |
+
Step 2780/44000 | Loss: 3.4972 | LR: 4.97e-04 | 1351.8ms/step | 14.8 samples/s
|
| 439 |
+
Step 2790/44000 | Loss: 3.4920 | LR: 4.97e-04 | 1355.4ms/step | 14.8 samples/s
|
| 440 |
+
Step 2800/44000 | Loss: 3.5118 | LR: 4.97e-04 | 1359.8ms/step | 14.7 samples/s
|
| 441 |
+
Step 2810/44000 | Loss: 3.4875 | LR: 4.97e-04 | 1358.3ms/step | 14.7 samples/s
|
| 442 |
+
Step 2820/44000 | Loss: 3.4969 | LR: 4.97e-04 | 1357.8ms/step | 14.7 samples/s
|
| 443 |
+
Step 2830/44000 | Loss: 3.4963 | LR: 4.97e-04 | 1359.5ms/step | 14.7 samples/s
|
| 444 |
+
Step 2840/44000 | Loss: 3.4830 | LR: 4.97e-04 | 1360.5ms/step | 14.7 samples/s
|
| 445 |
+
Step 2850/44000 | Loss: 3.4863 | LR: 4.97e-04 | 1359.6ms/step | 14.7 samples/s
|
epfl_final/train_logs/365m_edge_dense_altmixed_seed49_resume.log
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_205024
|
| 2 |
+
Running config:
|
| 3 |
+
{
|
| 4 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 5 |
+
"model_dtype": "bfloat16",
|
| 6 |
+
"init_from_scratch": true,
|
| 7 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 8 |
+
"val_path": null,
|
| 9 |
+
"max_seq_length": 8192,
|
| 10 |
+
"num_workers": 8,
|
| 11 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_205024",
|
| 12 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750",
|
| 13 |
+
"force_new_run": true,
|
| 14 |
+
"num_epochs": 1,
|
| 15 |
+
"max_steps": 44000,
|
| 16 |
+
"per_device_batch_size": 2,
|
| 17 |
+
"gradient_accumulation_steps": 10,
|
| 18 |
+
"learning_rate": 0.0005,
|
| 19 |
+
"weight_decay": 0.01,
|
| 20 |
+
"warmup_steps": 750,
|
| 21 |
+
"lr_scheduler": "cosine",
|
| 22 |
+
"max_grad_norm": 1.0,
|
| 23 |
+
"save_steps": 250,
|
| 24 |
+
"eval_steps": 250,
|
| 25 |
+
"eval_batches": 50,
|
| 26 |
+
"logging_steps": 10,
|
| 27 |
+
"seed": 49,
|
| 28 |
+
"verbose": false,
|
| 29 |
+
"checkpoint_activations": false,
|
| 30 |
+
"strategy": "ddp",
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"wandb_enabled": true,
|
| 33 |
+
"wandb_project": "stochastic_sparsity",
|
| 34 |
+
"wandb_name": "365m edge-dense alternating-mixed seed49",
|
| 35 |
+
"wandb_tags": [
|
| 36 |
+
"qwen3",
|
| 37 |
+
"365m",
|
| 38 |
+
"8k",
|
| 39 |
+
"edge-dense",
|
| 40 |
+
"alternating-mixed",
|
| 41 |
+
"seed49"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": null,
|
| 50 |
+
"rope_train_position_scale_set": null,
|
| 51 |
+
"attention_type": "sparse_stochastic",
|
| 52 |
+
"sparse_block_size": 128,
|
| 53 |
+
"sparse_num_random_blocks": 31,
|
| 54 |
+
"sparse_mask_pool_size": 10000,
|
| 55 |
+
"sparse_val_dense_compare": false,
|
| 56 |
+
"sparse_layer_indices": [
|
| 57 |
+
3,
|
| 58 |
+
5,
|
| 59 |
+
7,
|
| 60 |
+
9,
|
| 61 |
+
11,
|
| 62 |
+
13,
|
| 63 |
+
15,
|
| 64 |
+
17,
|
| 65 |
+
19,
|
| 66 |
+
21,
|
| 67 |
+
23
|
| 68 |
+
],
|
| 69 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 70 |
+
"sparse_mask_seed": 0,
|
| 71 |
+
"sparse_mixed_nrb_set": [
|
| 72 |
+
63,
|
| 73 |
+
47,
|
| 74 |
+
31,
|
| 75 |
+
21,
|
| 76 |
+
11
|
| 77 |
+
],
|
| 78 |
+
"dense_mode": false,
|
| 79 |
+
"dense_step_intervals": null
|
| 80 |
+
}
|
| 81 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 82 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750
|
| 83 |
+
Checkpoint activations: False
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
================================================================================
|
| 87 |
+
MODEL ARCHITECTURE
|
| 88 |
+
================================================================================
|
| 89 |
+
Model config:
|
| 90 |
+
vocab_size: 151936
|
| 91 |
+
hidden_size: 768
|
| 92 |
+
num_hidden_layers: 28
|
| 93 |
+
num_attention_heads: 6
|
| 94 |
+
num_key_value_heads: 2
|
| 95 |
+
head_dim: 128
|
| 96 |
+
intermediate_size: 3072
|
| 97 |
+
max_position_embeddings: 32768
|
| 98 |
+
rope_theta: 1000000
|
| 99 |
+
rope_train_position_scale_set: None
|
| 100 |
+
attention_type: sparse_stochastic
|
| 101 |
+
sparse_block_size: 128
|
| 102 |
+
sparse_num_random_blocks: 31
|
| 103 |
+
sparse_mask_pool_size: 10000
|
| 104 |
+
sparse_max_seq_length: 8192
|
| 105 |
+
|
| 106 |
+
Parameter count:
|
| 107 |
+
Total parameters: 358,958,848
|
| 108 |
+
Trainable parameters: 358,958,848
|
| 109 |
+
Non-trainable parameters: 0
|
| 110 |
+
Model size (BF16): 0.72 GB
|
| 111 |
+
================================================================================
|
| 112 |
+
|
| 113 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 114 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 115 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 116 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 117 |
+
Dataset split: 3027764 train, 159356 validation
|
| 118 |
+
Training examples (packed, train split): 3027764
|
| 119 |
+
Validation examples (packed, val split): 159356
|
| 120 |
+
Per-device batch size: 2
|
| 121 |
+
World size (GPUs): 1
|
| 122 |
+
Gradient accumulation steps: 10
|
| 123 |
+
Effective batch size (seqs/step): 20
|
| 124 |
+
Sequence length: 8192
|
| 125 |
+
Tokens per optimizer step: 163,840
|
| 126 |
+
Micro-batches per epoch (per rank): 1513882
|
| 127 |
+
Optimizer steps per epoch: 151388
|
| 128 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 129 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_edge_dense_altmixed_seed49/run_20260825_192725/checkpoint-2750...
|
| 130 |
+
Preserving checkpoint LR state for exact resume: current=[0.0004973664984850435], schedule_base=[0.0005]
|
| 131 |
+
Resumed from checkpoint at step 2750 (epoch 0, micro_step 27500)
|
| 132 |
+
Resuming training from step 2750
|
| 133 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 134 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 135 |
+
wandb: setting up run yl9g6w7y
|
| 136 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 137 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_205215-yl9g6w7y
|
| 138 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 139 |
+
wandb: Syncing run 365m edge-dense alternating-mixed seed49
|
| 140 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 141 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/yl9g6w7y
|
| 142 |
+
Starting training...
|
| 143 |
+
Validation at step 2750: Loss = 3.5117, Perplexity = 33.50
|
| 144 |
+
Switching attention mode for step 2751: sparse_stochastic (configured sparse attention)
|
| 145 |
+
Step 2760/44000 | Loss: 3.4835 | LR: 4.97e-04 | 19957.9ms/step | 1.0 samples/s
|
| 146 |
+
Step 2770/44000 | Loss: 3.5085 | LR: 4.97e-04 | 1379.4ms/step | 14.5 samples/s
|
| 147 |
+
Step 2780/44000 | Loss: 3.5160 | LR: 4.97e-04 | 1383.5ms/step | 14.5 samples/s
|
| 148 |
+
Step 2790/44000 | Loss: 3.5128 | LR: 4.97e-04 | 1386.7ms/step | 14.4 samples/s
|
epfl_final/train_logs/a100_40gb_vram_probe_p28.log
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832
|
| 2 |
+
Running config:
|
| 3 |
+
{
|
| 4 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 5 |
+
"model_dtype": "bfloat16",
|
| 6 |
+
"init_from_scratch": true,
|
| 7 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 8 |
+
"val_path": null,
|
| 9 |
+
"max_seq_length": 8192,
|
| 10 |
+
"num_workers": 8,
|
| 11 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832",
|
| 12 |
+
"resume_from_checkpoint": null,
|
| 13 |
+
"force_new_run": true,
|
| 14 |
+
"num_epochs": 1,
|
| 15 |
+
"max_steps": 2,
|
| 16 |
+
"per_device_batch_size": 2,
|
| 17 |
+
"gradient_accumulation_steps": 10,
|
| 18 |
+
"learning_rate": 0.0005,
|
| 19 |
+
"weight_decay": 0.01,
|
| 20 |
+
"warmup_steps": 750,
|
| 21 |
+
"lr_scheduler": "cosine",
|
| 22 |
+
"max_grad_norm": 1.0,
|
| 23 |
+
"save_steps": 999999,
|
| 24 |
+
"eval_steps": 999999,
|
| 25 |
+
"eval_batches": 50,
|
| 26 |
+
"logging_steps": 1,
|
| 27 |
+
"seed": 42,
|
| 28 |
+
"verbose": true,
|
| 29 |
+
"checkpoint_activations": false,
|
| 30 |
+
"strategy": "ddp",
|
| 31 |
+
"mixed_precision": "bf16",
|
| 32 |
+
"wandb_enabled": false,
|
| 33 |
+
"wandb_project": "stochastic_sparsity",
|
| 34 |
+
"wandb_name": "l8k365m_dense_dropout_p28_seed42",
|
| 35 |
+
"wandb_tags": [
|
| 36 |
+
"qwen3",
|
| 37 |
+
"365m",
|
| 38 |
+
"8k",
|
| 39 |
+
"scaleladder",
|
| 40 |
+
"dense"
|
| 41 |
+
],
|
| 42 |
+
"hidden_size": 768,
|
| 43 |
+
"intermediate_size": 3072,
|
| 44 |
+
"num_attention_heads": 6,
|
| 45 |
+
"num_key_value_heads": 2,
|
| 46 |
+
"num_hidden_layers": 28,
|
| 47 |
+
"head_dim": null,
|
| 48 |
+
"attention_dropout": 0.28,
|
| 49 |
+
"attention_type": "sparse_stochastic",
|
| 50 |
+
"sparse_block_size": 128,
|
| 51 |
+
"sparse_num_random_blocks": 31,
|
| 52 |
+
"sparse_mask_pool_size": 1,
|
| 53 |
+
"sparse_val_dense_compare": false,
|
| 54 |
+
"sparse_layer_indices": null,
|
| 55 |
+
"sparse_mask_cache_dir": null,
|
| 56 |
+
"sparse_mask_seed": 0,
|
| 57 |
+
"sparse_mixed_nrb_set": null,
|
| 58 |
+
"dense_mode": true,
|
| 59 |
+
"dense_step_intervals": null
|
| 60 |
+
}
|
| 61 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 62 |
+
Initializing model from scratch (random weights)...
|
| 63 |
+
Checkpoint activations: False
|
| 64 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
================================================================================
|
| 68 |
+
MODEL ARCHITECTURE
|
| 69 |
+
================================================================================
|
| 70 |
+
Model config:
|
| 71 |
+
vocab_size: 151936
|
| 72 |
+
hidden_size: 768
|
| 73 |
+
num_hidden_layers: 28
|
| 74 |
+
num_attention_heads: 6
|
| 75 |
+
num_key_value_heads: 2
|
| 76 |
+
head_dim: 128
|
| 77 |
+
intermediate_size: 3072
|
| 78 |
+
max_position_embeddings: 32768
|
| 79 |
+
rope_theta: 1000000
|
| 80 |
+
attention_type: sparse_stochastic
|
| 81 |
+
sparse_block_size: 128
|
| 82 |
+
sparse_num_random_blocks: 31
|
| 83 |
+
sparse_mask_pool_size: 1
|
| 84 |
+
sparse_max_seq_length: 8192
|
| 85 |
+
|
| 86 |
+
Parameter count:
|
| 87 |
+
Total parameters: 358,958,848
|
| 88 |
+
Trainable parameters: 358,958,848
|
| 89 |
+
Non-trainable parameters: 0
|
| 90 |
+
Model size (BF16): 0.72 GB
|
| 91 |
+
|
| 92 |
+
--------------------------------------------------------------------------------
|
| 93 |
+
LAYER-BY-LAYER SUMMARY
|
| 94 |
+
--------------------------------------------------------------------------------
|
| 95 |
+
================================================================================================================================================================
|
| 96 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 97 |
+
================================================================================================================================================================
|
| 98 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 99 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True
|
| 100 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True
|
| 101 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 102 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 103 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 104 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 105 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 106 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 107 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 108 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 109 |
+
��� │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 130 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True
|
| 131 |
+
├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True
|
| 132 |
+
================================================================================================================================================================
|
| 133 |
+
Total params: 475,645,696
|
| 134 |
+
Trainable params: 475,645,696
|
| 135 |
+
Non-trainable params: 0
|
| 136 |
+
Total mult-adds (Units.MEGABYTES): 951.29
|
| 137 |
+
================================================================================================================================================================
|
| 138 |
+
Input size (MB): 0.13
|
| 139 |
+
Forward/backward pass size (MB): 31197.23
|
| 140 |
+
Params size (MB): 951.29
|
| 141 |
+
Estimated Total Size (MB): 32148.66
|
| 142 |
+
================================================================================================================================================================
|
| 143 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
|
| 144 |
+
================================================================================
|
| 145 |
+
|
| 146 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 147 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 148 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 149 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 150 |
+
Dataset split: 3027764 train, 159356 validation
|
| 151 |
+
Training examples (packed, train split): 3027764
|
| 152 |
+
Validation examples (packed, val split): 159356
|
| 153 |
+
Per-device batch size: 2
|
| 154 |
+
World size (GPUs): 1
|
| 155 |
+
Gradient accumulation steps: 10
|
| 156 |
+
Effective batch size (seqs/step): 20
|
| 157 |
+
Sequence length: 8192
|
| 158 |
+
Tokens per optimizer step: 163,840
|
| 159 |
+
Micro-batches per epoch (per rank): 1513882
|
| 160 |
+
Optimizer steps per epoch: 151388
|
| 161 |
+
Total optimizer steps: 2 (capped by max_steps; 0.00 epochs)
|
| 162 |
+
Starting training...
|
| 163 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 164 |
+
Step 1/2 | Loss: 12.0550 | LR: 6.67e-07 | 92045.4ms/step | 0.2 samples/s
|
| 165 |
+
Step 2/2 | Loss: 12.0397 | LR: 1.33e-06 | 3736.5ms/step | 5.4 samples/s
|
| 166 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/a100_40gb_vram_probe_p28/run_20260819_142832/checkpoint-2
|
| 167 |
+
Training complete!
|
epfl_final/train_logs/dense_dropout_p28_8k365m_s43.run.log
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed43/run_20260819_050018
|
| 3 |
+
Running config:
|
| 4 |
+
{
|
| 5 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 6 |
+
"model_dtype": "bfloat16",
|
| 7 |
+
"init_from_scratch": true,
|
| 8 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 9 |
+
"val_path": null,
|
| 10 |
+
"max_seq_length": 8192,
|
| 11 |
+
"num_workers": 8,
|
| 12 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed43/run_20260819_050018",
|
| 13 |
+
"resume_from_checkpoint": null,
|
| 14 |
+
"force_new_run": true,
|
| 15 |
+
"num_epochs": 1,
|
| 16 |
+
"max_steps": 44000,
|
| 17 |
+
"per_device_batch_size": 2,
|
| 18 |
+
"gradient_accumulation_steps": 10,
|
| 19 |
+
"learning_rate": 0.0005,
|
| 20 |
+
"weight_decay": 0.01,
|
| 21 |
+
"warmup_steps": 750,
|
| 22 |
+
"lr_scheduler": "cosine",
|
| 23 |
+
"max_grad_norm": 1.0,
|
| 24 |
+
"save_steps": 1000,
|
| 25 |
+
"eval_steps": 1000,
|
| 26 |
+
"eval_batches": 50,
|
| 27 |
+
"logging_steps": 10,
|
| 28 |
+
"seed": 43,
|
| 29 |
+
"verbose": true,
|
| 30 |
+
"checkpoint_activations": false,
|
| 31 |
+
"strategy": "ddp",
|
| 32 |
+
"mixed_precision": "bf16",
|
| 33 |
+
"wandb_enabled": false,
|
| 34 |
+
"wandb_project": "stochastic_sparsity",
|
| 35 |
+
"wandb_name": "l8k365m_dense_dropout_p28_seed43",
|
| 36 |
+
"wandb_tags": [
|
| 37 |
+
"qwen3",
|
| 38 |
+
"365m",
|
| 39 |
+
"8k",
|
| 40 |
+
"scaleladder",
|
| 41 |
+
"dense"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": 0.28,
|
| 50 |
+
"attention_type": "sparse_stochastic",
|
| 51 |
+
"sparse_block_size": 128,
|
| 52 |
+
"sparse_num_random_blocks": 31,
|
| 53 |
+
"sparse_mask_pool_size": 1,
|
| 54 |
+
"sparse_val_dense_compare": false,
|
| 55 |
+
"sparse_layer_indices": null,
|
| 56 |
+
"sparse_mask_cache_dir": null,
|
| 57 |
+
"sparse_mask_seed": 0,
|
| 58 |
+
"sparse_mixed_nrb_set": null,
|
| 59 |
+
"dense_mode": true,
|
| 60 |
+
"dense_step_intervals": null
|
| 61 |
+
}
|
| 62 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 63 |
+
Initializing model from scratch (random weights)...
|
| 64 |
+
Checkpoint activations: False
|
| 65 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
================================================================================
|
| 69 |
+
MODEL ARCHITECTURE
|
| 70 |
+
================================================================================
|
| 71 |
+
Model config:
|
| 72 |
+
vocab_size: 151936
|
| 73 |
+
hidden_size: 768
|
| 74 |
+
num_hidden_layers: 28
|
| 75 |
+
num_attention_heads: 6
|
| 76 |
+
num_key_value_heads: 2
|
| 77 |
+
head_dim: 128
|
| 78 |
+
intermediate_size: 3072
|
| 79 |
+
max_position_embeddings: 32768
|
| 80 |
+
rope_theta: 1000000
|
| 81 |
+
attention_type: sparse_stochastic
|
| 82 |
+
sparse_block_size: 128
|
| 83 |
+
sparse_num_random_blocks: 31
|
| 84 |
+
sparse_mask_pool_size: 1
|
| 85 |
+
sparse_max_seq_length: 8192
|
| 86 |
+
|
| 87 |
+
Parameter count:
|
| 88 |
+
Total parameters: 358,958,848
|
| 89 |
+
Trainable parameters: 358,958,848
|
| 90 |
+
Non-trainable parameters: 0
|
| 91 |
+
Model size (BF16): 0.72 GB
|
| 92 |
+
|
| 93 |
+
--------------------------------------------------------------------------------
|
| 94 |
+
LAYER-BY-LAYER SUMMARY
|
| 95 |
+
--------------------------------------------------------------------------------
|
| 96 |
+
================================================================================================================================================================
|
| 97 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 98 |
+
================================================================================================================================================================
|
| 99 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 100 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True
|
| 101 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True
|
| 102 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 103 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 104 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 105 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 106 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 107 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 108 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 109 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 131 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True
|
| 132 |
+
├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True
|
| 133 |
+
================================================================================================================================================================
|
| 134 |
+
Total params: 475,645,696
|
| 135 |
+
Trainable params: 475,645,696
|
| 136 |
+
Non-trainable params: 0
|
| 137 |
+
Total mult-adds (Units.MEGABYTES): 951.29
|
| 138 |
+
================================================================================================================================================================
|
| 139 |
+
Input size (MB): 0.13
|
| 140 |
+
Forward/backward pass size (MB): 31197.23
|
| 141 |
+
Params size (MB): 951.29
|
| 142 |
+
Estimated Total Size (MB): 32148.66
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
|
| 145 |
+
================================================================================
|
| 146 |
+
|
| 147 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 148 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 149 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 150 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 151 |
+
Dataset split: 3027764 train, 159356 validation
|
| 152 |
+
Training examples (packed, train split): 3027764
|
| 153 |
+
Validation examples (packed, val split): 159356
|
| 154 |
+
Per-device batch size: 2
|
| 155 |
+
World size (GPUs): 1
|
| 156 |
+
Gradient accumulation steps: 10
|
| 157 |
+
Effective batch size (seqs/step): 20
|
| 158 |
+
Sequence length: 8192
|
| 159 |
+
Tokens per optimizer step: 163,840
|
| 160 |
+
Micro-batches per epoch (per rank): 1513882
|
| 161 |
+
Optimizer steps per epoch: 151388
|
| 162 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 163 |
+
Starting training...
|
| 164 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 165 |
+
Step 10/44000 | Loss: 12.0463 | LR: 6.67e-06 | 7508.3ms/step | 2.7 samples/s
|
| 166 |
+
Step 20/44000 | Loss: 11.9585 | LR: 1.33e-05 | 1547.9ms/step | 12.9 samples/s
|
| 167 |
+
Step 30/44000 | Loss: 11.6537 | LR: 2.00e-05 | 1550.0ms/step | 12.9 samples/s
|
| 168 |
+
Step 40/44000 | Loss: 11.1215 | LR: 2.67e-05 | 1545.1ms/step | 12.9 samples/s
|
| 169 |
+
Step 50/44000 | Loss: 10.7750 | LR: 3.33e-05 | 1543.7ms/step | 13.0 samples/s
|
| 170 |
+
Step 60/44000 | Loss: 10.5377 | LR: 4.00e-05 | 1542.6ms/step | 13.0 samples/s
|
| 171 |
+
Step 70/44000 | Loss: 10.2935 | LR: 4.67e-05 | 1541.1ms/step | 13.0 samples/s
|
| 172 |
+
Step 80/44000 | Loss: 9.9947 | LR: 5.33e-05 | 1539.3ms/step | 13.0 samples/s
|
| 173 |
+
Step 90/44000 | Loss: 9.6406 | LR: 6.00e-05 | 1539.4ms/step | 13.0 samples/s
|
| 174 |
+
Step 100/44000 | Loss: 9.1423 | LR: 6.67e-05 | 1542.6ms/step | 13.0 samples/s
|
| 175 |
+
Step 110/44000 | Loss: 8.6275 | LR: 7.33e-05 | 1543.3ms/step | 13.0 samples/s
|
| 176 |
+
Step 120/44000 | Loss: 8.1296 | LR: 8.00e-05 | 1543.1ms/step | 13.0 samples/s
|
| 177 |
+
Step 130/44000 | Loss: 7.7712 | LR: 8.67e-05 | 1543.2ms/step | 13.0 samples/s
|
| 178 |
+
Step 140/44000 | Loss: 7.5022 | LR: 9.33e-05 | 1543.2ms/step | 13.0 samples/s
|
| 179 |
+
Step 150/44000 | Loss: 7.3504 | LR: 1.00e-04 | 1544.7ms/step | 12.9 samples/s
|
| 180 |
+
Step 160/44000 | Loss: 7.2178 | LR: 1.07e-04 | 1545.6ms/step | 12.9 samples/s
|
| 181 |
+
Step 170/44000 | Loss: 7.1287 | LR: 1.13e-04 | 1549.8ms/step | 12.9 samples/s
|
| 182 |
+
Step 180/44000 | Loss: 7.0587 | LR: 1.20e-04 | 1550.5ms/step | 12.9 samples/s
|
| 183 |
+
Step 190/44000 | Loss: 6.9497 | LR: 1.27e-04 | 1552.1ms/step | 12.9 samples/s
|
| 184 |
+
Step 200/44000 | Loss: 6.8464 | LR: 1.33e-04 | 1552.5ms/step | 12.9 samples/s
|
| 185 |
+
Step 210/44000 | Loss: 6.7536 | LR: 1.40e-04 | 1556.4ms/step | 12.8 samples/s
|
| 186 |
+
Step 220/44000 | Loss: 6.6817 | LR: 1.47e-04 | 1558.9ms/step | 12.8 samples/s
|
| 187 |
+
Step 230/44000 | Loss: 6.5950 | LR: 1.53e-04 | 1557.5ms/step | 12.8 samples/s
|
| 188 |
+
Step 240/44000 | Loss: 6.5252 | LR: 1.60e-04 | 1555.3ms/step | 12.9 samples/s
|
| 189 |
+
Step 250/44000 | Loss: 6.4606 | LR: 1.67e-04 | 1559.5ms/step | 12.8 samples/s
|
| 190 |
+
Step 260/44000 | Loss: 6.4260 | LR: 1.73e-04 | 1560.9ms/step | 12.8 samples/s
|
| 191 |
+
Step 270/44000 | Loss: 6.3791 | LR: 1.80e-04 | 1557.4ms/step | 12.8 samples/s
|
| 192 |
+
Step 280/44000 | Loss: 6.3171 | LR: 1.87e-04 | 1558.3ms/step | 12.8 samples/s
|
| 193 |
+
Step 290/44000 | Loss: 6.2716 | LR: 1.93e-04 | 1555.7ms/step | 12.9 samples/s
|
| 194 |
+
Step 300/44000 | Loss: 6.2159 | LR: 2.00e-04 | 1558.7ms/step | 12.8 samples/s
|
| 195 |
+
Step 310/44000 | Loss: 6.1887 | LR: 2.07e-04 | 1558.9ms/step | 12.8 samples/s
|
| 196 |
+
Step 320/44000 | Loss: 6.1583 | LR: 2.13e-04 | 1555.7ms/step | 12.9 samples/s
|
| 197 |
+
Step 330/44000 | Loss: 6.1042 | LR: 2.20e-04 | 1559.9ms/step | 12.8 samples/s
|
| 198 |
+
Step 340/44000 | Loss: 6.0994 | LR: 2.27e-04 | 1560.4ms/step | 12.8 samples/s
|
| 199 |
+
Step 350/44000 | Loss: 6.0454 | LR: 2.33e-04 | 1559.4ms/step | 12.8 samples/s
|
| 200 |
+
Step 360/44000 | Loss: 5.9798 | LR: 2.40e-04 | 1561.7ms/step | 12.8 samples/s
|
| 201 |
+
Step 370/44000 | Loss: 5.9713 | LR: 2.47e-04 | 1561.2ms/step | 12.8 samples/s
|
| 202 |
+
Step 380/44000 | Loss: 5.9319 | LR: 2.53e-04 | 1561.4ms/step | 12.8 samples/s
|
| 203 |
+
Step 390/44000 | Loss: 5.8644 | LR: 2.60e-04 | 1560.4ms/step | 12.8 samples/s
|
| 204 |
+
Step 400/44000 | Loss: 5.8794 | LR: 2.67e-04 | 1563.9ms/step | 12.8 samples/s
|
| 205 |
+
Step 410/44000 | Loss: 5.8098 | LR: 2.73e-04 | 1562.7ms/step | 12.8 samples/s
|
| 206 |
+
Step 420/44000 | Loss: 5.7697 | LR: 2.80e-04 | 1561.2ms/step | 12.8 samples/s
|
| 207 |
+
Step 430/44000 | Loss: 5.7735 | LR: 2.87e-04 | 1560.0ms/step | 12.8 samples/s
|
| 208 |
+
Step 440/44000 | Loss: 5.7114 | LR: 2.93e-04 | 1561.1ms/step | 12.8 samples/s
|
| 209 |
+
Step 450/44000 | Loss: 5.6882 | LR: 3.00e-04 | 1563.4ms/step | 12.8 samples/s
|
| 210 |
+
Step 460/44000 | Loss: 5.6444 | LR: 3.07e-04 | 1561.0ms/step | 12.8 samples/s
|
| 211 |
+
Step 470/44000 | Loss: 5.6119 | LR: 3.13e-04 | 1561.6ms/step | 12.8 samples/s
|
| 212 |
+
Step 480/44000 | Loss: 5.6104 | LR: 3.20e-04 | 1558.9ms/step | 12.8 samples/s
|
| 213 |
+
Step 490/44000 | Loss: 5.5278 | LR: 3.27e-04 | 1560.4ms/step | 12.8 samples/s
|
| 214 |
+
Step 500/44000 | Loss: 5.5088 | LR: 3.33e-04 | 1564.1ms/step | 12.8 samples/s
|
| 215 |
+
Step 510/44000 | Loss: 5.5104 | LR: 3.40e-04 | 1559.4ms/step | 12.8 samples/s
|
| 216 |
+
Step 520/44000 | Loss: 5.4609 | LR: 3.47e-04 | 1561.5ms/step | 12.8 samples/s
|
| 217 |
+
Step 530/44000 | Loss: 5.4333 | LR: 3.53e-04 | 1558.3ms/step | 12.8 samples/s
|
| 218 |
+
Step 540/44000 | Loss: 5.4356 | LR: 3.60e-04 | 1561.1ms/step | 12.8 samples/s
|
| 219 |
+
Step 550/44000 | Loss: 5.3679 | LR: 3.67e-04 | 1562.4ms/step | 12.8 samples/s
|
| 220 |
+
Step 560/44000 | Loss: 5.3378 | LR: 3.73e-04 | 1560.5ms/step | 12.8 samples/s
|
| 221 |
+
Step 570/44000 | Loss: 5.3173 | LR: 3.80e-04 | 1562.4ms/step | 12.8 samples/s
|
| 222 |
+
Step 580/44000 | Loss: 5.3159 | LR: 3.87e-04 | 1560.2ms/step | 12.8 samples/s
|
| 223 |
+
Step 590/44000 | Loss: 5.2586 | LR: 3.93e-04 | 1557.9ms/step | 12.8 samples/s
|
| 224 |
+
Step 600/44000 | Loss: 5.2425 | LR: 4.00e-04 | 1559.5ms/step | 12.8 samples/s
|
| 225 |
+
Step 610/44000 | Loss: 5.2087 | LR: 4.07e-04 | 1559.7ms/step | 12.8 samples/s
|
| 226 |
+
Step 620/44000 | Loss: 5.2016 | LR: 4.13e-04 | 1560.9ms/step | 12.8 samples/s
|
| 227 |
+
Step 630/44000 | Loss: 5.1308 | LR: 4.20e-04 | 1565.8ms/step | 12.8 samples/s
|
| 228 |
+
Step 640/44000 | Loss: 5.0982 | LR: 4.27e-04 | 1562.0ms/step | 12.8 samples/s
|
| 229 |
+
Step 650/44000 | Loss: 5.1127 | LR: 4.33e-04 | 1560.3ms/step | 12.8 samples/s
|
| 230 |
+
Step 660/44000 | Loss: 5.0744 | LR: 4.40e-04 | 1563.5ms/step | 12.8 samples/s
|
| 231 |
+
Step 670/44000 | Loss: 5.0514 | LR: 4.47e-04 | 1560.4ms/step | 12.8 samples/s
|
| 232 |
+
Step 680/44000 | Loss: 5.0014 | LR: 4.53e-04 | 1561.4ms/step | 12.8 samples/s
|
| 233 |
+
Step 690/44000 | Loss: 5.0109 | LR: 4.60e-04 | 1560.3ms/step | 12.8 samples/s
|
| 234 |
+
Step 700/44000 | Loss: 4.9526 | LR: 4.67e-04 | 1561.8ms/step | 12.8 samples/s
|
| 235 |
+
Step 710/44000 | Loss: 4.9058 | LR: 4.73e-04 | 1559.9ms/step | 12.8 samples/s
|
| 236 |
+
Step 720/44000 | Loss: 4.9391 | LR: 4.80e-04 | 1559.7ms/step | 12.8 samples/s
|
| 237 |
+
Step 730/44000 | Loss: 4.8740 | LR: 4.87e-04 | 1559.0ms/step | 12.8 samples/s
|
| 238 |
+
Step 740/44000 | Loss: 4.8697 | LR: 4.93e-04 | 1558.4ms/step | 12.8 samples/s
|
| 239 |
+
Step 750/44000 | Loss: 4.8330 | LR: 5.00e-04 | 1556.8ms/step | 12.8 samples/s
|
| 240 |
+
Step 760/44000 | Loss: 4.8444 | LR: 5.00e-04 | 1557.9ms/step | 12.8 samples/s
|
| 241 |
+
Step 770/44000 | Loss: 4.7579 | LR: 5.00e-04 | 1560.2ms/step | 12.8 samples/s
|
| 242 |
+
Step 780/44000 | Loss: 4.7678 | LR: 5.00e-04 | 1561.7ms/step | 12.8 samples/s
|
| 243 |
+
Step 790/44000 | Loss: 4.6798 | LR: 5.00e-04 | 1560.0ms/step | 12.8 samples/s
|
| 244 |
+
Step 800/44000 | Loss: 4.6436 | LR: 5.00e-04 | 1560.5ms/step | 12.8 samples/s
|
| 245 |
+
Step 810/44000 | Loss: 4.6535 | LR: 5.00e-04 | 1559.4ms/step | 12.8 samples/s
|
| 246 |
+
Step 820/44000 | Loss: 4.6358 | LR: 5.00e-04 | 1562.5ms/step | 12.8 samples/s
|
| 247 |
+
Step 830/44000 | Loss: 4.5948 | LR: 5.00e-04 | 1557.8ms/step | 12.8 samples/s
|
| 248 |
+
Step 840/44000 | Loss: 4.5725 | LR: 5.00e-04 | 1557.7ms/step | 12.8 samples/s
|
| 249 |
+
Step 850/44000 | Loss: 4.5451 | LR: 5.00e-04 | 1560.1ms/step | 12.8 samples/s
|
| 250 |
+
Step 860/44000 | Loss: 4.5535 | LR: 5.00e-04 | 1560.7ms/step | 12.8 samples/s
|
| 251 |
+
Step 870/44000 | Loss: 4.5061 | LR: 5.00e-04 | 1561.4ms/step | 12.8 samples/s
|
| 252 |
+
Step 880/44000 | Loss: 4.5126 | LR: 5.00e-04 | 1562.1ms/step | 12.8 samples/s
|
| 253 |
+
Step 890/44000 | Loss: 4.4986 | LR: 5.00e-04 | 1559.9ms/step | 12.8 samples/s
|
| 254 |
+
Step 900/44000 | Loss: 4.4784 | LR: 5.00e-04 | 1563.5ms/step | 12.8 samples/s
|
| 255 |
+
Step 910/44000 | Loss: 4.4291 | LR: 5.00e-04 | 1560.1ms/step | 12.8 samples/s
|
| 256 |
+
Step 920/44000 | Loss: 4.4476 | LR: 5.00e-04 | 1561.6ms/step | 12.8 samples/s
|
| 257 |
+
Step 930/44000 | Loss: 4.4132 | LR: 5.00e-04 | 1557.9ms/step | 12.8 samples/s
|
| 258 |
+
Step 940/44000 | Loss: 4.3968 | LR: 5.00e-04 | 1559.8ms/step | 12.8 samples/s
|
| 259 |
+
Step 950/44000 | Loss: 4.4018 | LR: 5.00e-04 | 1556.7ms/step | 12.8 samples/s
|
| 260 |
+
Step 960/44000 | Loss: 4.3777 | LR: 5.00e-04 | 1558.0ms/step | 12.8 samples/s
|
| 261 |
+
Step 970/44000 | Loss: 4.3747 | LR: 5.00e-04 | 1558.6ms/step | 12.8 samples/s
|
| 262 |
+
Step 980/44000 | Loss: 4.3335 | LR: 5.00e-04 | 1559.7ms/step | 12.8 samples/s
|
| 263 |
+
Step 990/44000 | Loss: 4.3208 | LR: 5.00e-04 | 1559.5ms/step | 12.8 samples/s
|
| 264 |
+
Step 1000/44000 | Loss: 4.3212 | LR: 5.00e-04 | 1557.0ms/step | 12.8 samples/s
|
epfl_final/train_logs/dense_dropout_p28_8k365m_s44.run.log
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed44/run_20260819_050018
|
| 3 |
+
Running config:
|
| 4 |
+
{
|
| 5 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 6 |
+
"model_dtype": "bfloat16",
|
| 7 |
+
"init_from_scratch": true,
|
| 8 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 9 |
+
"val_path": null,
|
| 10 |
+
"max_seq_length": 8192,
|
| 11 |
+
"num_workers": 8,
|
| 12 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed44/run_20260819_050018",
|
| 13 |
+
"resume_from_checkpoint": null,
|
| 14 |
+
"force_new_run": true,
|
| 15 |
+
"num_epochs": 1,
|
| 16 |
+
"max_steps": 44000,
|
| 17 |
+
"per_device_batch_size": 2,
|
| 18 |
+
"gradient_accumulation_steps": 10,
|
| 19 |
+
"learning_rate": 0.0005,
|
| 20 |
+
"weight_decay": 0.01,
|
| 21 |
+
"warmup_steps": 750,
|
| 22 |
+
"lr_scheduler": "cosine",
|
| 23 |
+
"max_grad_norm": 1.0,
|
| 24 |
+
"save_steps": 1000,
|
| 25 |
+
"eval_steps": 1000,
|
| 26 |
+
"eval_batches": 50,
|
| 27 |
+
"logging_steps": 10,
|
| 28 |
+
"seed": 44,
|
| 29 |
+
"verbose": true,
|
| 30 |
+
"checkpoint_activations": false,
|
| 31 |
+
"strategy": "ddp",
|
| 32 |
+
"mixed_precision": "bf16",
|
| 33 |
+
"wandb_enabled": false,
|
| 34 |
+
"wandb_project": "stochastic_sparsity",
|
| 35 |
+
"wandb_name": "l8k365m_dense_dropout_p28_seed44",
|
| 36 |
+
"wandb_tags": [
|
| 37 |
+
"qwen3",
|
| 38 |
+
"365m",
|
| 39 |
+
"8k",
|
| 40 |
+
"scaleladder",
|
| 41 |
+
"dense"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": 0.28,
|
| 50 |
+
"attention_type": "sparse_stochastic",
|
| 51 |
+
"sparse_block_size": 128,
|
| 52 |
+
"sparse_num_random_blocks": 31,
|
| 53 |
+
"sparse_mask_pool_size": 1,
|
| 54 |
+
"sparse_val_dense_compare": false,
|
| 55 |
+
"sparse_layer_indices": null,
|
| 56 |
+
"sparse_mask_cache_dir": null,
|
| 57 |
+
"sparse_mask_seed": 0,
|
| 58 |
+
"sparse_mixed_nrb_set": null,
|
| 59 |
+
"dense_mode": true,
|
| 60 |
+
"dense_step_intervals": null
|
| 61 |
+
}
|
| 62 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 63 |
+
Initializing model from scratch (random weights)...
|
| 64 |
+
Checkpoint activations: False
|
| 65 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
================================================================================
|
| 69 |
+
MODEL ARCHITECTURE
|
| 70 |
+
================================================================================
|
| 71 |
+
Model config:
|
| 72 |
+
vocab_size: 151936
|
| 73 |
+
hidden_size: 768
|
| 74 |
+
num_hidden_layers: 28
|
| 75 |
+
num_attention_heads: 6
|
| 76 |
+
num_key_value_heads: 2
|
| 77 |
+
head_dim: 128
|
| 78 |
+
intermediate_size: 3072
|
| 79 |
+
max_position_embeddings: 32768
|
| 80 |
+
rope_theta: 1000000
|
| 81 |
+
attention_type: sparse_stochastic
|
| 82 |
+
sparse_block_size: 128
|
| 83 |
+
sparse_num_random_blocks: 31
|
| 84 |
+
sparse_mask_pool_size: 1
|
| 85 |
+
sparse_max_seq_length: 8192
|
| 86 |
+
|
| 87 |
+
Parameter count:
|
| 88 |
+
Total parameters: 358,958,848
|
| 89 |
+
Trainable parameters: 358,958,848
|
| 90 |
+
Non-trainable parameters: 0
|
| 91 |
+
Model size (BF16): 0.72 GB
|
| 92 |
+
|
| 93 |
+
--------------------------------------------------------------------------------
|
| 94 |
+
LAYER-BY-LAYER SUMMARY
|
| 95 |
+
--------------------------------------------------------------------------------
|
| 96 |
+
================================================================================================================================================================
|
| 97 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 98 |
+
================================================================================================================================================================
|
| 99 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 100 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True
|
| 101 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True
|
| 102 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 103 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 104 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 105 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 106 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 107 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 108 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 109 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 131 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True
|
| 132 |
+
├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True
|
| 133 |
+
================================================================================================================================================================
|
| 134 |
+
Total params: 475,645,696
|
| 135 |
+
Trainable params: 475,645,696
|
| 136 |
+
Non-trainable params: 0
|
| 137 |
+
Total mult-adds (Units.MEGABYTES): 951.29
|
| 138 |
+
================================================================================================================================================================
|
| 139 |
+
Input size (MB): 0.13
|
| 140 |
+
Forward/backward pass size (MB): 31197.23
|
| 141 |
+
Params size (MB): 951.29
|
| 142 |
+
Estimated Total Size (MB): 32148.66
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
|
| 145 |
+
================================================================================
|
| 146 |
+
|
| 147 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 148 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 149 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 150 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 151 |
+
Dataset split: 3027764 train, 159356 validation
|
| 152 |
+
Training examples (packed, train split): 3027764
|
| 153 |
+
Validation examples (packed, val split): 159356
|
| 154 |
+
Per-device batch size: 2
|
| 155 |
+
World size (GPUs): 1
|
| 156 |
+
Gradient accumulation steps: 10
|
| 157 |
+
Effective batch size (seqs/step): 20
|
| 158 |
+
Sequence length: 8192
|
| 159 |
+
Tokens per optimizer step: 163,840
|
| 160 |
+
Micro-batches per epoch (per rank): 1513882
|
| 161 |
+
Optimizer steps per epoch: 151388
|
| 162 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 163 |
+
Starting training...
|
| 164 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 165 |
+
Step 10/44000 | Loss: 12.0416 | LR: 6.67e-06 | 7505.1ms/step | 2.7 samples/s
|
| 166 |
+
Step 20/44000 | Loss: 11.9527 | LR: 1.33e-05 | 1540.2ms/step | 13.0 samples/s
|
| 167 |
+
Step 30/44000 | Loss: 11.6367 | LR: 2.00e-05 | 1540.4ms/step | 13.0 samples/s
|
| 168 |
+
Step 40/44000 | Loss: 11.1027 | LR: 2.67e-05 | 1537.1ms/step | 13.0 samples/s
|
| 169 |
+
Step 50/44000 | Loss: 10.7707 | LR: 3.33e-05 | 1535.7ms/step | 13.0 samples/s
|
| 170 |
+
Step 60/44000 | Loss: 10.5164 | LR: 4.00e-05 | 1536.6ms/step | 13.0 samples/s
|
| 171 |
+
Step 70/44000 | Loss: 10.2922 | LR: 4.67e-05 | 1534.5ms/step | 13.0 samples/s
|
| 172 |
+
Step 80/44000 | Loss: 9.9943 | LR: 5.33e-05 | 1533.0ms/step | 13.0 samples/s
|
| 173 |
+
Step 90/44000 | Loss: 9.6171 | LR: 6.00e-05 | 1535.4ms/step | 13.0 samples/s
|
| 174 |
+
Step 100/44000 | Loss: 9.1483 | LR: 6.67e-05 | 1536.9ms/step | 13.0 samples/s
|
| 175 |
+
Step 110/44000 | Loss: 8.6214 | LR: 7.33e-05 | 1536.0ms/step | 13.0 samples/s
|
| 176 |
+
Step 120/44000 | Loss: 8.1305 | LR: 8.00e-05 | 1535.0ms/step | 13.0 samples/s
|
| 177 |
+
Step 130/44000 | Loss: 7.7622 | LR: 8.67e-05 | 1535.4ms/step | 13.0 samples/s
|
| 178 |
+
Step 140/44000 | Loss: 7.5438 | LR: 9.33e-05 | 1537.6ms/step | 13.0 samples/s
|
| 179 |
+
Step 150/44000 | Loss: 7.3373 | LR: 1.00e-04 | 1538.6ms/step | 13.0 samples/s
|
| 180 |
+
Step 160/44000 | Loss: 7.2345 | LR: 1.07e-04 | 1540.6ms/step | 13.0 samples/s
|
| 181 |
+
Step 170/44000 | Loss: 7.1003 | LR: 1.13e-04 | 1544.6ms/step | 12.9 samples/s
|
| 182 |
+
Step 180/44000 | Loss: 7.0074 | LR: 1.20e-04 | 1543.5ms/step | 13.0 samples/s
|
| 183 |
+
Step 190/44000 | Loss: 6.8941 | LR: 1.27e-04 | 1545.2ms/step | 12.9 samples/s
|
| 184 |
+
Step 200/44000 | Loss: 6.8372 | LR: 1.33e-04 | 1544.5ms/step | 12.9 samples/s
|
| 185 |
+
Step 210/44000 | Loss: 6.7709 | LR: 1.40e-04 | 1542.8ms/step | 13.0 samples/s
|
| 186 |
+
Step 220/44000 | Loss: 6.6943 | LR: 1.47e-04 | 1545.2ms/step | 12.9 samples/s
|
| 187 |
+
Step 230/44000 | Loss: 6.6290 | LR: 1.53e-04 | 1545.6ms/step | 12.9 samples/s
|
| 188 |
+
Step 240/44000 | Loss: 6.5209 | LR: 1.60e-04 | 1542.8ms/step | 13.0 samples/s
|
| 189 |
+
Step 250/44000 | Loss: 6.4791 | LR: 1.67e-04 | 1546.7ms/step | 12.9 samples/s
|
| 190 |
+
Step 260/44000 | Loss: 6.4193 | LR: 1.73e-04 | 1548.3ms/step | 12.9 samples/s
|
| 191 |
+
Step 270/44000 | Loss: 6.3734 | LR: 1.80e-04 | 1548.0ms/step | 12.9 samples/s
|
| 192 |
+
Step 280/44000 | Loss: 6.3138 | LR: 1.87e-04 | 1547.5ms/step | 12.9 samples/s
|
| 193 |
+
Step 290/44000 | Loss: 6.2986 | LR: 1.93e-04 | 1546.2ms/step | 12.9 samples/s
|
| 194 |
+
Step 300/44000 | Loss: 6.2422 | LR: 2.00e-04 | 1548.0ms/step | 12.9 samples/s
|
| 195 |
+
Step 310/44000 | Loss: 6.2184 | LR: 2.07e-04 | 1549.2ms/step | 12.9 samples/s
|
| 196 |
+
Step 320/44000 | Loss: 6.1643 | LR: 2.13e-04 | 1550.7ms/step | 12.9 samples/s
|
| 197 |
+
Step 330/44000 | Loss: 6.1384 | LR: 2.20e-04 | 1547.6ms/step | 12.9 samples/s
|
| 198 |
+
Step 340/44000 | Loss: 6.0679 | LR: 2.27e-04 | 1549.2ms/step | 12.9 samples/s
|
| 199 |
+
Step 350/44000 | Loss: 6.0403 | LR: 2.33e-04 | 1552.2ms/step | 12.9 samples/s
|
| 200 |
+
Step 360/44000 | Loss: 5.9892 | LR: 2.40e-04 | 1549.3ms/step | 12.9 samples/s
|
| 201 |
+
Step 370/44000 | Loss: 5.9561 | LR: 2.47e-04 | 1550.5ms/step | 12.9 samples/s
|
| 202 |
+
Step 380/44000 | Loss: 5.9219 | LR: 2.53e-04 | 1550.5ms/step | 12.9 samples/s
|
| 203 |
+
Step 390/44000 | Loss: 5.8810 | LR: 2.60e-04 | 1547.5ms/step | 12.9 samples/s
|
| 204 |
+
Step 400/44000 | Loss: 5.8826 | LR: 2.67e-04 | 1551.3ms/step | 12.9 samples/s
|
| 205 |
+
Step 410/44000 | Loss: 5.8343 | LR: 2.73e-04 | 1549.2ms/step | 12.9 samples/s
|
| 206 |
+
Step 420/44000 | Loss: 5.7985 | LR: 2.80e-04 | 1550.6ms/step | 12.9 samples/s
|
| 207 |
+
Step 430/44000 | Loss: 5.8005 | LR: 2.87e-04 | 1550.7ms/step | 12.9 samples/s
|
| 208 |
+
Step 440/44000 | Loss: 5.7497 | LR: 2.93e-04 | 1551.7ms/step | 12.9 samples/s
|
| 209 |
+
Step 450/44000 | Loss: 5.6865 | LR: 3.00e-04 | 1550.4ms/step | 12.9 samples/s
|
| 210 |
+
Step 460/44000 | Loss: 5.6686 | LR: 3.07e-04 | 1550.7ms/step | 12.9 samples/s
|
| 211 |
+
Step 470/44000 | Loss: 5.6188 | LR: 3.13e-04 | 1548.6ms/step | 12.9 samples/s
|
| 212 |
+
Step 480/44000 | Loss: 5.5638 | LR: 3.20e-04 | 1552.5ms/step | 12.9 samples/s
|
| 213 |
+
Step 490/44000 | Loss: 5.5654 | LR: 3.27e-04 | 1551.7ms/step | 12.9 samples/s
|
| 214 |
+
Step 500/44000 | Loss: 5.5286 | LR: 3.33e-04 | 1550.2ms/step | 12.9 samples/s
|
| 215 |
+
Step 510/44000 | Loss: 5.4902 | LR: 3.40e-04 | 1548.7ms/step | 12.9 samples/s
|
| 216 |
+
Step 520/44000 | Loss: 5.4674 | LR: 3.47e-04 | 1550.3ms/step | 12.9 samples/s
|
| 217 |
+
Step 530/44000 | Loss: 5.4335 | LR: 3.53e-04 | 1549.2ms/step | 12.9 samples/s
|
| 218 |
+
Step 540/44000 | Loss: 5.3960 | LR: 3.60e-04 | 1548.7ms/step | 12.9 samples/s
|
| 219 |
+
Step 550/44000 | Loss: 5.4036 | LR: 3.67e-04 | 1552.7ms/step | 12.9 samples/s
|
| 220 |
+
Step 560/44000 | Loss: 5.3486 | LR: 3.73e-04 | 1549.7ms/step | 12.9 samples/s
|
| 221 |
+
Step 570/44000 | Loss: 5.3179 | LR: 3.80e-04 | 1548.4ms/step | 12.9 samples/s
|
| 222 |
+
Step 580/44000 | Loss: 5.2564 | LR: 3.87e-04 | 1550.4ms/step | 12.9 samples/s
|
| 223 |
+
Step 590/44000 | Loss: 5.2692 | LR: 3.93e-04 | 1549.4ms/step | 12.9 samples/s
|
| 224 |
+
Step 600/44000 | Loss: 5.2297 | LR: 4.00e-04 | 1547.3ms/step | 12.9 samples/s
|
| 225 |
+
Step 610/44000 | Loss: 5.2292 | LR: 4.07e-04 | 1548.1ms/step | 12.9 samples/s
|
| 226 |
+
Step 620/44000 | Loss: 5.1748 | LR: 4.13e-04 | 1550.5ms/step | 12.9 samples/s
|
| 227 |
+
Step 630/44000 | Loss: 5.1369 | LR: 4.20e-04 | 1548.8ms/step | 12.9 samples/s
|
| 228 |
+
Step 640/44000 | Loss: 5.1490 | LR: 4.27e-04 | 1548.7ms/step | 12.9 samples/s
|
| 229 |
+
Step 650/44000 | Loss: 5.0831 | LR: 4.33e-04 | 1549.5ms/step | 12.9 samples/s
|
| 230 |
+
Step 660/44000 | Loss: 5.0300 | LR: 4.40e-04 | 1551.1ms/step | 12.9 samples/s
|
| 231 |
+
Step 670/44000 | Loss: 5.0499 | LR: 4.47e-04 | 1551.2ms/step | 12.9 samples/s
|
| 232 |
+
Step 680/44000 | Loss: 5.0349 | LR: 4.53e-04 | 1551.7ms/step | 12.9 samples/s
|
| 233 |
+
Step 690/44000 | Loss: 4.9896 | LR: 4.60e-04 | 1551.8ms/step | 12.9 samples/s
|
| 234 |
+
Step 700/44000 | Loss: 4.9603 | LR: 4.67e-04 | 1549.6ms/step | 12.9 samples/s
|
| 235 |
+
Step 710/44000 | Loss: 4.9015 | LR: 4.73e-04 | 1551.2ms/step | 12.9 samples/s
|
| 236 |
+
Step 720/44000 | Loss: 4.9155 | LR: 4.80e-04 | 1550.7ms/step | 12.9 samples/s
|
| 237 |
+
Step 730/44000 | Loss: 4.9029 | LR: 4.87e-04 | 1550.3ms/step | 12.9 samples/s
|
| 238 |
+
Step 740/44000 | Loss: 4.8483 | LR: 4.93e-04 | 1545.7ms/step | 12.9 samples/s
|
| 239 |
+
Step 750/44000 | Loss: 4.7587 | LR: 5.00e-04 | 1545.8ms/step | 12.9 samples/s
|
| 240 |
+
Step 760/44000 | Loss: 4.8035 | LR: 5.00e-04 | 1544.9ms/step | 12.9 samples/s
|
| 241 |
+
Step 770/44000 | Loss: 4.7709 | LR: 5.00e-04 | 1547.2ms/step | 12.9 samples/s
|
| 242 |
+
Step 780/44000 | Loss: 4.7268 | LR: 5.00e-04 | 1548.1ms/step | 12.9 samples/s
|
| 243 |
+
Step 790/44000 | Loss: 4.7146 | LR: 5.00e-04 | 1548.6ms/step | 12.9 samples/s
|
| 244 |
+
Step 800/44000 | Loss: 4.7016 | LR: 5.00e-04 | 1546.9ms/step | 12.9 samples/s
|
| 245 |
+
Step 810/44000 | Loss: 4.6566 | LR: 5.00e-04 | 1547.2ms/step | 12.9 samples/s
|
| 246 |
+
Step 820/44000 | Loss: 4.6499 | LR: 5.00e-04 | 1549.1ms/step | 12.9 samples/s
|
| 247 |
+
Step 830/44000 | Loss: 4.6243 | LR: 5.00e-04 | 1549.4ms/step | 12.9 samples/s
|
| 248 |
+
Step 840/44000 | Loss: 4.5618 | LR: 5.00e-04 | 1548.9ms/step | 12.9 samples/s
|
| 249 |
+
Step 850/44000 | Loss: 4.5523 | LR: 5.00e-04 | 1545.6ms/step | 12.9 samples/s
|
| 250 |
+
Step 860/44000 | Loss: 4.5522 | LR: 5.00e-04 | 1548.2ms/step | 12.9 samples/s
|
| 251 |
+
Step 870/44000 | Loss: 4.5004 | LR: 5.00e-04 | 1549.4ms/step | 12.9 samples/s
|
| 252 |
+
Step 880/44000 | Loss: 4.5067 | LR: 5.00e-04 | 1549.6ms/step | 12.9 samples/s
|
| 253 |
+
Step 890/44000 | Loss: 4.4939 | LR: 5.00e-04 | 1549.0ms/step | 12.9 samples/s
|
| 254 |
+
Step 900/44000 | Loss: 4.4681 | LR: 5.00e-04 | 1549.1ms/step | 12.9 samples/s
|
| 255 |
+
Step 910/44000 | Loss: 4.4435 | LR: 5.00e-04 | 1547.5ms/step | 12.9 samples/s
|
| 256 |
+
Step 920/44000 | Loss: 4.4134 | LR: 5.00e-04 | 1546.9ms/step | 12.9 samples/s
|
| 257 |
+
Step 930/44000 | Loss: 4.4117 | LR: 5.00e-04 | 1548.2ms/step | 12.9 samples/s
|
| 258 |
+
Step 940/44000 | Loss: 4.4234 | LR: 5.00e-04 | 1549.0ms/step | 12.9 samples/s
|
| 259 |
+
Step 950/44000 | Loss: 4.3647 | LR: 5.00e-04 | 1546.9ms/step | 12.9 samples/s
|
| 260 |
+
Step 960/44000 | Loss: 4.3981 | LR: 5.00e-04 | 1546.1ms/step | 12.9 samples/s
|
| 261 |
+
Step 970/44000 | Loss: 4.3631 | LR: 5.00e-04 | 1548.2ms/step | 12.9 samples/s
|
| 262 |
+
Step 980/44000 | Loss: 4.3378 | LR: 5.00e-04 | 1547.8ms/step | 12.9 samples/s
|
| 263 |
+
Step 990/44000 | Loss: 4.3385 | LR: 5.00e-04 | 1548.1ms/step | 12.9 samples/s
|
| 264 |
+
Step 1000/44000 | Loss: 4.3095 | LR: 5.00e-04 | 1546.1ms/step | 12.9 samples/s
|
epfl_final/train_logs/dense_dropout_p28_8k365m_s45.run.log
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed45/run_20260819_050018
|
| 3 |
+
Running config:
|
| 4 |
+
{
|
| 5 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 6 |
+
"model_dtype": "bfloat16",
|
| 7 |
+
"init_from_scratch": true,
|
| 8 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 9 |
+
"val_path": null,
|
| 10 |
+
"max_seq_length": 8192,
|
| 11 |
+
"num_workers": 8,
|
| 12 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed45/run_20260819_050018",
|
| 13 |
+
"resume_from_checkpoint": null,
|
| 14 |
+
"force_new_run": true,
|
| 15 |
+
"num_epochs": 1,
|
| 16 |
+
"max_steps": 44000,
|
| 17 |
+
"per_device_batch_size": 2,
|
| 18 |
+
"gradient_accumulation_steps": 10,
|
| 19 |
+
"learning_rate": 0.0005,
|
| 20 |
+
"weight_decay": 0.01,
|
| 21 |
+
"warmup_steps": 750,
|
| 22 |
+
"lr_scheduler": "cosine",
|
| 23 |
+
"max_grad_norm": 1.0,
|
| 24 |
+
"save_steps": 1000,
|
| 25 |
+
"eval_steps": 1000,
|
| 26 |
+
"eval_batches": 50,
|
| 27 |
+
"logging_steps": 10,
|
| 28 |
+
"seed": 45,
|
| 29 |
+
"verbose": true,
|
| 30 |
+
"checkpoint_activations": false,
|
| 31 |
+
"strategy": "ddp",
|
| 32 |
+
"mixed_precision": "bf16",
|
| 33 |
+
"wandb_enabled": false,
|
| 34 |
+
"wandb_project": "stochastic_sparsity",
|
| 35 |
+
"wandb_name": "l8k365m_dense_dropout_p28_seed45",
|
| 36 |
+
"wandb_tags": [
|
| 37 |
+
"qwen3",
|
| 38 |
+
"365m",
|
| 39 |
+
"8k",
|
| 40 |
+
"scaleladder",
|
| 41 |
+
"dense"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": 0.28,
|
| 50 |
+
"attention_type": "sparse_stochastic",
|
| 51 |
+
"sparse_block_size": 128,
|
| 52 |
+
"sparse_num_random_blocks": 31,
|
| 53 |
+
"sparse_mask_pool_size": 1,
|
| 54 |
+
"sparse_val_dense_compare": false,
|
| 55 |
+
"sparse_layer_indices": null,
|
| 56 |
+
"sparse_mask_cache_dir": null,
|
| 57 |
+
"sparse_mask_seed": 0,
|
| 58 |
+
"sparse_mixed_nrb_set": null,
|
| 59 |
+
"dense_mode": true,
|
| 60 |
+
"dense_step_intervals": null
|
| 61 |
+
}
|
| 62 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 63 |
+
Initializing model from scratch (random weights)...
|
| 64 |
+
Checkpoint activations: False
|
| 65 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
================================================================================
|
| 69 |
+
MODEL ARCHITECTURE
|
| 70 |
+
================================================================================
|
| 71 |
+
Model config:
|
| 72 |
+
vocab_size: 151936
|
| 73 |
+
hidden_size: 768
|
| 74 |
+
num_hidden_layers: 28
|
| 75 |
+
num_attention_heads: 6
|
| 76 |
+
num_key_value_heads: 2
|
| 77 |
+
head_dim: 128
|
| 78 |
+
intermediate_size: 3072
|
| 79 |
+
max_position_embeddings: 32768
|
| 80 |
+
rope_theta: 1000000
|
| 81 |
+
attention_type: sparse_stochastic
|
| 82 |
+
sparse_block_size: 128
|
| 83 |
+
sparse_num_random_blocks: 31
|
| 84 |
+
sparse_mask_pool_size: 1
|
| 85 |
+
sparse_max_seq_length: 8192
|
| 86 |
+
|
| 87 |
+
Parameter count:
|
| 88 |
+
Total parameters: 358,958,848
|
| 89 |
+
Trainable parameters: 358,958,848
|
| 90 |
+
Non-trainable parameters: 0
|
| 91 |
+
Model size (BF16): 0.72 GB
|
| 92 |
+
|
| 93 |
+
--------------------------------------------------------------------------------
|
| 94 |
+
LAYER-BY-LAYER SUMMARY
|
| 95 |
+
--------------------------------------------------------------------------------
|
| 96 |
+
================================================================================================================================================================
|
| 97 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 98 |
+
================================================================================================================================================================
|
| 99 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 100 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True
|
| 101 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True
|
| 102 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 103 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 104 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 105 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 106 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 107 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 108 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 109 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 131 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True
|
| 132 |
+
├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True
|
| 133 |
+
================================================================================================================================================================
|
| 134 |
+
Total params: 475,645,696
|
| 135 |
+
Trainable params: 475,645,696
|
| 136 |
+
Non-trainable params: 0
|
| 137 |
+
Total mult-adds (Units.MEGABYTES): 951.29
|
| 138 |
+
================================================================================================================================================================
|
| 139 |
+
Input size (MB): 0.13
|
| 140 |
+
Forward/backward pass size (MB): 31197.23
|
| 141 |
+
Params size (MB): 951.29
|
| 142 |
+
Estimated Total Size (MB): 32148.66
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
|
| 145 |
+
================================================================================
|
| 146 |
+
|
| 147 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 148 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 149 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 150 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 151 |
+
Dataset split: 3027764 train, 159356 validation
|
| 152 |
+
Training examples (packed, train split): 3027764
|
| 153 |
+
Validation examples (packed, val split): 159356
|
| 154 |
+
Per-device batch size: 2
|
| 155 |
+
World size (GPUs): 1
|
| 156 |
+
Gradient accumulation steps: 10
|
| 157 |
+
Effective batch size (seqs/step): 20
|
| 158 |
+
Sequence length: 8192
|
| 159 |
+
Tokens per optimizer step: 163,840
|
| 160 |
+
Micro-batches per epoch (per rank): 1513882
|
| 161 |
+
Optimizer steps per epoch: 151388
|
| 162 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 163 |
+
Starting training...
|
| 164 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 165 |
+
Step 10/44000 | Loss: 12.0712 | LR: 6.67e-06 | 7510.1ms/step | 2.7 samples/s
|
| 166 |
+
Step 20/44000 | Loss: 11.9859 | LR: 1.33e-05 | 1550.1ms/step | 12.9 samples/s
|
| 167 |
+
Step 30/44000 | Loss: 11.6848 | LR: 2.00e-05 | 1548.2ms/step | 12.9 samples/s
|
| 168 |
+
Step 40/44000 | Loss: 11.1722 | LR: 2.67e-05 | 1546.7ms/step | 12.9 samples/s
|
| 169 |
+
Step 50/44000 | Loss: 10.8348 | LR: 3.33e-05 | 1547.2ms/step | 12.9 samples/s
|
| 170 |
+
Step 60/44000 | Loss: 10.5909 | LR: 4.00e-05 | 1550.6ms/step | 12.9 samples/s
|
| 171 |
+
Step 70/44000 | Loss: 10.3500 | LR: 4.67e-05 | 1550.5ms/step | 12.9 samples/s
|
| 172 |
+
Step 80/44000 | Loss: 10.0516 | LR: 5.33e-05 | 1548.1ms/step | 12.9 samples/s
|
| 173 |
+
Step 90/44000 | Loss: 9.6944 | LR: 6.00e-05 | 1548.9ms/step | 12.9 samples/s
|
| 174 |
+
Step 100/44000 | Loss: 9.2010 | LR: 6.67e-05 | 1551.1ms/step | 12.9 samples/s
|
| 175 |
+
Step 110/44000 | Loss: 8.6517 | LR: 7.33e-05 | 1550.4ms/step | 12.9 samples/s
|
| 176 |
+
Step 120/44000 | Loss: 8.1723 | LR: 8.00e-05 | 1546.7ms/step | 12.9 samples/s
|
| 177 |
+
Step 130/44000 | Loss: 7.7978 | LR: 8.67e-05 | 1550.2ms/step | 12.9 samples/s
|
| 178 |
+
Step 140/44000 | Loss: 7.5260 | LR: 9.33e-05 | 1550.3ms/step | 12.9 samples/s
|
| 179 |
+
Step 150/44000 | Loss: 7.3593 | LR: 1.00e-04 | 1552.7ms/step | 12.9 samples/s
|
| 180 |
+
Step 160/44000 | Loss: 7.2156 | LR: 1.07e-04 | 1552.5ms/step | 12.9 samples/s
|
| 181 |
+
Step 170/44000 | Loss: 7.1290 | LR: 1.13e-04 | 1554.7ms/step | 12.9 samples/s
|
| 182 |
+
Step 180/44000 | Loss: 7.0328 | LR: 1.20e-04 | 1557.4ms/step | 12.8 samples/s
|
| 183 |
+
Step 190/44000 | Loss: 6.9560 | LR: 1.27e-04 | 1559.2ms/step | 12.8 samples/s
|
| 184 |
+
Step 200/44000 | Loss: 6.8580 | LR: 1.33e-04 | 1556.8ms/step | 12.8 samples/s
|
| 185 |
+
Step 210/44000 | Loss: 6.7719 | LR: 1.40e-04 | 1557.4ms/step | 12.8 samples/s
|
| 186 |
+
Step 220/44000 | Loss: 6.6763 | LR: 1.47e-04 | 1557.0ms/step | 12.8 samples/s
|
| 187 |
+
Step 230/44000 | Loss: 6.6151 | LR: 1.53e-04 | 1559.4ms/step | 12.8 samples/s
|
| 188 |
+
Step 240/44000 | Loss: 6.5672 | LR: 1.60e-04 | 1558.2ms/step | 12.8 samples/s
|
| 189 |
+
Step 250/44000 | Loss: 6.5001 | LR: 1.67e-04 | 1562.6ms/step | 12.8 samples/s
|
| 190 |
+
Step 260/44000 | Loss: 6.4523 | LR: 1.73e-04 | 1562.4ms/step | 12.8 samples/s
|
| 191 |
+
Step 270/44000 | Loss: 6.3705 | LR: 1.80e-04 | 1561.3ms/step | 12.8 samples/s
|
| 192 |
+
Step 280/44000 | Loss: 6.3637 | LR: 1.87e-04 | 1562.1ms/step | 12.8 samples/s
|
| 193 |
+
Step 290/44000 | Loss: 6.2887 | LR: 1.93e-04 | 1560.5ms/step | 12.8 samples/s
|
| 194 |
+
Step 300/44000 | Loss: 6.2305 | LR: 2.00e-04 | 1561.1ms/step | 12.8 samples/s
|
| 195 |
+
Step 310/44000 | Loss: 6.2181 | LR: 2.07e-04 | 1565.8ms/step | 12.8 samples/s
|
| 196 |
+
Step 320/44000 | Loss: 6.1721 | LR: 2.13e-04 | 1563.1ms/step | 12.8 samples/s
|
| 197 |
+
Step 330/44000 | Loss: 6.1118 | LR: 2.20e-04 | 1564.9ms/step | 12.8 samples/s
|
| 198 |
+
Step 340/44000 | Loss: 6.0476 | LR: 2.27e-04 | 1564.7ms/step | 12.8 samples/s
|
| 199 |
+
Step 350/44000 | Loss: 6.0601 | LR: 2.33e-04 | 1565.1ms/step | 12.8 samples/s
|
| 200 |
+
Step 360/44000 | Loss: 6.0119 | LR: 2.40e-04 | 1569.7ms/step | 12.7 samples/s
|
| 201 |
+
Step 370/44000 | Loss: 5.9515 | LR: 2.47e-04 | 1566.8ms/step | 12.8 samples/s
|
| 202 |
+
Step 380/44000 | Loss: 5.9401 | LR: 2.53e-04 | 1567.1ms/step | 12.8 samples/s
|
| 203 |
+
Step 390/44000 | Loss: 5.9133 | LR: 2.60e-04 | 1566.1ms/step | 12.8 samples/s
|
| 204 |
+
Step 400/44000 | Loss: 5.8795 | LR: 2.67e-04 | 1568.0ms/step | 12.8 samples/s
|
| 205 |
+
Step 410/44000 | Loss: 5.8342 | LR: 2.73e-04 | 1568.0ms/step | 12.8 samples/s
|
| 206 |
+
Step 420/44000 | Loss: 5.8303 | LR: 2.80e-04 | 1568.7ms/step | 12.7 samples/s
|
| 207 |
+
Step 430/44000 | Loss: 5.7448 | LR: 2.87e-04 | 1568.1ms/step | 12.8 samples/s
|
| 208 |
+
Step 440/44000 | Loss: 5.7319 | LR: 2.93e-04 | 1570.1ms/step | 12.7 samples/s
|
| 209 |
+
Step 450/44000 | Loss: 5.7130 | LR: 3.00e-04 | 1568.9ms/step | 12.7 samples/s
|
| 210 |
+
Step 460/44000 | Loss: 5.6703 | LR: 3.07e-04 | 1566.3ms/step | 12.8 samples/s
|
| 211 |
+
Step 470/44000 | Loss: 5.6358 | LR: 3.13e-04 | 1568.3ms/step | 12.8 samples/s
|
| 212 |
+
Step 480/44000 | Loss: 5.5805 | LR: 3.20e-04 | 1566.8ms/step | 12.8 samples/s
|
| 213 |
+
Step 490/44000 | Loss: 5.5864 | LR: 3.27e-04 | 1566.7ms/step | 12.8 samples/s
|
| 214 |
+
Step 500/44000 | Loss: 5.5390 | LR: 3.33e-04 | 1568.2ms/step | 12.8 samples/s
|
| 215 |
+
Step 510/44000 | Loss: 5.4614 | LR: 3.40e-04 | 1568.5ms/step | 12.8 samples/s
|
| 216 |
+
Step 520/44000 | Loss: 5.4709 | LR: 3.47e-04 | 1568.0ms/step | 12.8 samples/s
|
| 217 |
+
Step 530/44000 | Loss: 5.4313 | LR: 3.53e-04 | 1564.6ms/step | 12.8 samples/s
|
| 218 |
+
Step 540/44000 | Loss: 5.4122 | LR: 3.60e-04 | 1567.9ms/step | 12.8 samples/s
|
| 219 |
+
Step 550/44000 | Loss: 5.3722 | LR: 3.67e-04 | 1569.4ms/step | 12.7 samples/s
|
| 220 |
+
Step 560/44000 | Loss: 5.3614 | LR: 3.73e-04 | 1568.8ms/step | 12.7 samples/s
|
| 221 |
+
Step 570/44000 | Loss: 5.3291 | LR: 3.80e-04 | 1566.8ms/step | 12.8 samples/s
|
| 222 |
+
Step 580/44000 | Loss: 5.3115 | LR: 3.87e-04 | 1565.2ms/step | 12.8 samples/s
|
| 223 |
+
Step 590/44000 | Loss: 5.2409 | LR: 3.93e-04 | 1565.6ms/step | 12.8 samples/s
|
| 224 |
+
Step 600/44000 | Loss: 5.2160 | LR: 4.00e-04 | 1567.9ms/step | 12.8 samples/s
|
| 225 |
+
Step 610/44000 | Loss: 5.1703 | LR: 4.07e-04 | 1567.4ms/step | 12.8 samples/s
|
| 226 |
+
Step 620/44000 | Loss: 5.1858 | LR: 4.13e-04 | 1568.2ms/step | 12.8 samples/s
|
| 227 |
+
Step 630/44000 | Loss: 5.1472 | LR: 4.20e-04 | 1568.7ms/step | 12.7 samples/s
|
| 228 |
+
Step 640/44000 | Loss: 5.1306 | LR: 4.27e-04 | 1569.6ms/step | 12.7 samples/s
|
| 229 |
+
Step 650/44000 | Loss: 5.1007 | LR: 4.33e-04 | 1566.1ms/step | 12.8 samples/s
|
| 230 |
+
Step 660/44000 | Loss: 5.0913 | LR: 4.40e-04 | 1566.6ms/step | 12.8 samples/s
|
| 231 |
+
Step 670/44000 | Loss: 5.0534 | LR: 4.47e-04 | 1567.4ms/step | 12.8 samples/s
|
| 232 |
+
Step 680/44000 | Loss: 5.0018 | LR: 4.53e-04 | 1566.3ms/step | 12.8 samples/s
|
| 233 |
+
Step 690/44000 | Loss: 4.9443 | LR: 4.60e-04 | 1562.8ms/step | 12.8 samples/s
|
| 234 |
+
Step 700/44000 | Loss: 4.9707 | LR: 4.67e-04 | 1564.1ms/step | 12.8 samples/s
|
| 235 |
+
Step 710/44000 | Loss: 4.9130 | LR: 4.73e-04 | 1567.5ms/step | 12.8 samples/s
|
| 236 |
+
Step 720/44000 | Loss: 4.8848 | LR: 4.80e-04 | 1562.2ms/step | 12.8 samples/s
|
| 237 |
+
Step 730/44000 | Loss: 4.8453 | LR: 4.87e-04 | 1561.9ms/step | 12.8 samples/s
|
| 238 |
+
Step 740/44000 | Loss: 4.8564 | LR: 4.93e-04 | 1565.1ms/step | 12.8 samples/s
|
| 239 |
+
Step 750/44000 | Loss: 4.7855 | LR: 5.00e-04 | 1560.4ms/step | 12.8 samples/s
|
| 240 |
+
Step 760/44000 | Loss: 4.7760 | LR: 5.00e-04 | 1560.2ms/step | 12.8 samples/s
|
| 241 |
+
Step 770/44000 | Loss: 4.7547 | LR: 5.00e-04 | 1561.4ms/step | 12.8 samples/s
|
| 242 |
+
Step 780/44000 | Loss: 4.7316 | LR: 5.00e-04 | 1564.3ms/step | 12.8 samples/s
|
| 243 |
+
Step 790/44000 | Loss: 4.6785 | LR: 5.00e-04 | 1562.3ms/step | 12.8 samples/s
|
| 244 |
+
Step 800/44000 | Loss: 4.6600 | LR: 5.00e-04 | 1566.1ms/step | 12.8 samples/s
|
| 245 |
+
Step 810/44000 | Loss: 4.6729 | LR: 5.00e-04 | 1564.1ms/step | 12.8 samples/s
|
| 246 |
+
Step 820/44000 | Loss: 4.6323 | LR: 5.00e-04 | 1564.3ms/step | 12.8 samples/s
|
| 247 |
+
Step 830/44000 | Loss: 4.6105 | LR: 5.00e-04 | 1560.0ms/step | 12.8 samples/s
|
| 248 |
+
Step 840/44000 | Loss: 4.5832 | LR: 5.00e-04 | 1559.5ms/step | 12.8 samples/s
|
| 249 |
+
Step 850/44000 | Loss: 4.5379 | LR: 5.00e-04 | 1563.9ms/step | 12.8 samples/s
|
| 250 |
+
Step 860/44000 | Loss: 4.5368 | LR: 5.00e-04 | 1566.4ms/step | 12.8 samples/s
|
| 251 |
+
Step 870/44000 | Loss: 4.5133 | LR: 5.00e-04 | 1562.2ms/step | 12.8 samples/s
|
| 252 |
+
Step 880/44000 | Loss: 4.5084 | LR: 5.00e-04 | 1562.9ms/step | 12.8 samples/s
|
| 253 |
+
Step 890/44000 | Loss: 4.4940 | LR: 5.00e-04 | 1563.3ms/step | 12.8 samples/s
|
| 254 |
+
Step 900/44000 | Loss: 4.4383 | LR: 5.00e-04 | 1558.8ms/step | 12.8 samples/s
|
| 255 |
+
Step 910/44000 | Loss: 4.4524 | LR: 5.00e-04 | 1565.0ms/step | 12.8 samples/s
|
| 256 |
+
Step 920/44000 | Loss: 4.4046 | LR: 5.00e-04 | 1566.6ms/step | 12.8 samples/s
|
| 257 |
+
Step 930/44000 | Loss: 4.3995 | LR: 5.00e-04 | 1561.1ms/step | 12.8 samples/s
|
| 258 |
+
Step 940/44000 | Loss: 4.3719 | LR: 5.00e-04 | 1559.9ms/step | 12.8 samples/s
|
| 259 |
+
Step 950/44000 | Loss: 4.3699 | LR: 5.00e-04 | 1561.2ms/step | 12.8 samples/s
|
| 260 |
+
Step 960/44000 | Loss: 4.4055 | LR: 5.00e-04 | 1561.7ms/step | 12.8 samples/s
|
| 261 |
+
Step 970/44000 | Loss: 4.3854 | LR: 5.00e-04 | 1562.3ms/step | 12.8 samples/s
|
| 262 |
+
Step 980/44000 | Loss: 4.3344 | LR: 5.00e-04 | 1566.1ms/step | 12.8 samples/s
|
| 263 |
+
Step 990/44000 | Loss: 4.3312 | LR: 5.00e-04 | 1558.9ms/step | 12.8 samples/s
|
| 264 |
+
Step 1000/44000 | Loss: 4.2805 | LR: 5.00e-04 | 1560.2ms/step | 12.8 samples/s
|
epfl_final/train_logs/dense_dropout_p28_8k365m_s46.run.log
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed46/run_20260819_050018
|
| 3 |
+
Running config:
|
| 4 |
+
{
|
| 5 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 6 |
+
"model_dtype": "bfloat16",
|
| 7 |
+
"init_from_scratch": true,
|
| 8 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 9 |
+
"val_path": null,
|
| 10 |
+
"max_seq_length": 8192,
|
| 11 |
+
"num_workers": 8,
|
| 12 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed46/run_20260819_050018",
|
| 13 |
+
"resume_from_checkpoint": null,
|
| 14 |
+
"force_new_run": true,
|
| 15 |
+
"num_epochs": 1,
|
| 16 |
+
"max_steps": 44000,
|
| 17 |
+
"per_device_batch_size": 2,
|
| 18 |
+
"gradient_accumulation_steps": 10,
|
| 19 |
+
"learning_rate": 0.0005,
|
| 20 |
+
"weight_decay": 0.01,
|
| 21 |
+
"warmup_steps": 750,
|
| 22 |
+
"lr_scheduler": "cosine",
|
| 23 |
+
"max_grad_norm": 1.0,
|
| 24 |
+
"save_steps": 1000,
|
| 25 |
+
"eval_steps": 1000,
|
| 26 |
+
"eval_batches": 50,
|
| 27 |
+
"logging_steps": 10,
|
| 28 |
+
"seed": 46,
|
| 29 |
+
"verbose": true,
|
| 30 |
+
"checkpoint_activations": false,
|
| 31 |
+
"strategy": "ddp",
|
| 32 |
+
"mixed_precision": "bf16",
|
| 33 |
+
"wandb_enabled": false,
|
| 34 |
+
"wandb_project": "stochastic_sparsity",
|
| 35 |
+
"wandb_name": "l8k365m_dense_dropout_p28_seed46",
|
| 36 |
+
"wandb_tags": [
|
| 37 |
+
"qwen3",
|
| 38 |
+
"365m",
|
| 39 |
+
"8k",
|
| 40 |
+
"scaleladder",
|
| 41 |
+
"dense"
|
| 42 |
+
],
|
| 43 |
+
"hidden_size": 768,
|
| 44 |
+
"intermediate_size": 3072,
|
| 45 |
+
"num_attention_heads": 6,
|
| 46 |
+
"num_key_value_heads": 2,
|
| 47 |
+
"num_hidden_layers": 28,
|
| 48 |
+
"head_dim": null,
|
| 49 |
+
"attention_dropout": 0.28,
|
| 50 |
+
"attention_type": "sparse_stochastic",
|
| 51 |
+
"sparse_block_size": 128,
|
| 52 |
+
"sparse_num_random_blocks": 31,
|
| 53 |
+
"sparse_mask_pool_size": 1,
|
| 54 |
+
"sparse_val_dense_compare": false,
|
| 55 |
+
"sparse_layer_indices": null,
|
| 56 |
+
"sparse_mask_cache_dir": null,
|
| 57 |
+
"sparse_mask_seed": 0,
|
| 58 |
+
"sparse_mixed_nrb_set": null,
|
| 59 |
+
"dense_mode": true,
|
| 60 |
+
"dense_step_intervals": null
|
| 61 |
+
}
|
| 62 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 63 |
+
Initializing model from scratch (random weights)...
|
| 64 |
+
Checkpoint activations: False
|
| 65 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
================================================================================
|
| 69 |
+
MODEL ARCHITECTURE
|
| 70 |
+
================================================================================
|
| 71 |
+
Model config:
|
| 72 |
+
vocab_size: 151936
|
| 73 |
+
hidden_size: 768
|
| 74 |
+
num_hidden_layers: 28
|
| 75 |
+
num_attention_heads: 6
|
| 76 |
+
num_key_value_heads: 2
|
| 77 |
+
head_dim: 128
|
| 78 |
+
intermediate_size: 3072
|
| 79 |
+
max_position_embeddings: 32768
|
| 80 |
+
rope_theta: 1000000
|
| 81 |
+
attention_type: sparse_stochastic
|
| 82 |
+
sparse_block_size: 128
|
| 83 |
+
sparse_num_random_blocks: 31
|
| 84 |
+
sparse_mask_pool_size: 1
|
| 85 |
+
sparse_max_seq_length: 8192
|
| 86 |
+
|
| 87 |
+
Parameter count:
|
| 88 |
+
Total parameters: 358,958,848
|
| 89 |
+
Trainable parameters: 358,958,848
|
| 90 |
+
Non-trainable parameters: 0
|
| 91 |
+
Model size (BF16): 0.72 GB
|
| 92 |
+
|
| 93 |
+
--------------------------------------------------------------------------------
|
| 94 |
+
LAYER-BY-LAYER SUMMARY
|
| 95 |
+
--------------------------------------------------------------------------------
|
| 96 |
+
================================================================================================================================================================
|
| 97 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 98 |
+
================================================================================================================================================================
|
| 99 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 100 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 768] -- True
|
| 101 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 768] 116,686,848 True
|
| 102 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 103 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 104 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 105 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 106 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 107 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 108 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 109 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 768] [2, 8192, 768] 8,652,544 True
|
| 131 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 768] [2, 8192, 768] 768 True
|
| 132 |
+
├─Linear: 1-2 [2, 8192, 768] [2, 8192, 151936] 116,686,848 True
|
| 133 |
+
================================================================================================================================================================
|
| 134 |
+
Total params: 475,645,696
|
| 135 |
+
Trainable params: 475,645,696
|
| 136 |
+
Non-trainable params: 0
|
| 137 |
+
Total mult-adds (Units.MEGABYTES): 951.29
|
| 138 |
+
================================================================================================================================================================
|
| 139 |
+
Input size (MB): 0.13
|
| 140 |
+
Forward/backward pass size (MB): 31197.23
|
| 141 |
+
Params size (MB): 951.29
|
| 142 |
+
Estimated Total Size (MB): 32148.66
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (116,686,848 params). True unique parameter count: 358,958,848 (359.0M)
|
| 145 |
+
================================================================================
|
| 146 |
+
|
| 147 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 148 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 149 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 150 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 151 |
+
Dataset split: 3027764 train, 159356 validation
|
| 152 |
+
Training examples (packed, train split): 3027764
|
| 153 |
+
Validation examples (packed, val split): 159356
|
| 154 |
+
Per-device batch size: 2
|
| 155 |
+
World size (GPUs): 1
|
| 156 |
+
Gradient accumulation steps: 10
|
| 157 |
+
Effective batch size (seqs/step): 20
|
| 158 |
+
Sequence length: 8192
|
| 159 |
+
Tokens per optimizer step: 163,840
|
| 160 |
+
Micro-batches per epoch (per rank): 1513882
|
| 161 |
+
Optimizer steps per epoch: 151388
|
| 162 |
+
Total optimizer steps: 44000 (capped by max_steps; 0.29 epochs)
|
| 163 |
+
Starting training...
|
| 164 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 165 |
+
Step 10/44000 | Loss: 12.0573 | LR: 6.67e-06 | 7503.2ms/step | 2.7 samples/s
|
| 166 |
+
Step 20/44000 | Loss: 11.9633 | LR: 1.33e-05 | 1537.1ms/step | 13.0 samples/s
|
| 167 |
+
Step 30/44000 | Loss: 11.6441 | LR: 2.00e-05 | 1534.4ms/step | 13.0 samples/s
|
| 168 |
+
Step 40/44000 | Loss: 11.1096 | LR: 2.67e-05 | 1530.0ms/step | 13.1 samples/s
|
| 169 |
+
Step 50/44000 | Loss: 10.7761 | LR: 3.33e-05 | 1533.6ms/step | 13.0 samples/s
|
| 170 |
+
Step 60/44000 | Loss: 10.5566 | LR: 4.00e-05 | 1530.2ms/step | 13.1 samples/s
|
| 171 |
+
Step 70/44000 | Loss: 10.2996 | LR: 4.67e-05 | 1527.2ms/step | 13.1 samples/s
|
| 172 |
+
Step 80/44000 | Loss: 10.0009 | LR: 5.33e-05 | 1528.7ms/step | 13.1 samples/s
|
| 173 |
+
Step 90/44000 | Loss: 9.5833 | LR: 6.00e-05 | 1531.8ms/step | 13.1 samples/s
|
| 174 |
+
Step 100/44000 | Loss: 9.1600 | LR: 6.67e-05 | 1530.4ms/step | 13.1 samples/s
|
| 175 |
+
Step 110/44000 | Loss: 8.6192 | LR: 7.33e-05 | 1530.3ms/step | 13.1 samples/s
|
| 176 |
+
Step 120/44000 | Loss: 8.1617 | LR: 8.00e-05 | 1534.0ms/step | 13.0 samples/s
|
| 177 |
+
Step 130/44000 | Loss: 7.7907 | LR: 8.67e-05 | 1530.8ms/step | 13.1 samples/s
|
| 178 |
+
Step 140/44000 | Loss: 7.5197 | LR: 9.33e-05 | 1529.8ms/step | 13.1 samples/s
|
| 179 |
+
Step 150/44000 | Loss: 7.3554 | LR: 1.00e-04 | 1535.6ms/step | 13.0 samples/s
|
| 180 |
+
Step 160/44000 | Loss: 7.2174 | LR: 1.07e-04 | 1535.7ms/step | 13.0 samples/s
|
| 181 |
+
Step 170/44000 | Loss: 7.1398 | LR: 1.13e-04 | 1534.1ms/step | 13.0 samples/s
|
| 182 |
+
Step 180/44000 | Loss: 7.0423 | LR: 1.20e-04 | 1537.7ms/step | 13.0 samples/s
|
| 183 |
+
Step 190/44000 | Loss: 6.9235 | LR: 1.27e-04 | 1540.3ms/step | 13.0 samples/s
|
| 184 |
+
Step 200/44000 | Loss: 6.8622 | LR: 1.33e-04 | 1536.8ms/step | 13.0 samples/s
|
| 185 |
+
Step 210/44000 | Loss: 6.7649 | LR: 1.40e-04 | 1535.9ms/step | 13.0 samples/s
|
| 186 |
+
Step 220/44000 | Loss: 6.6905 | LR: 1.47e-04 | 1540.3ms/step | 13.0 samples/s
|
| 187 |
+
Step 230/44000 | Loss: 6.6202 | LR: 1.53e-04 | 1540.0ms/step | 13.0 samples/s
|
| 188 |
+
Step 240/44000 | Loss: 6.5612 | LR: 1.60e-04 | 1536.0ms/step | 13.0 samples/s
|
| 189 |
+
Step 250/44000 | Loss: 6.5372 | LR: 1.67e-04 | 1539.4ms/step | 13.0 samples/s
|
| 190 |
+
Step 260/44000 | Loss: 6.4226 | LR: 1.73e-04 | 1541.1ms/step | 13.0 samples/s
|
| 191 |
+
Step 270/44000 | Loss: 6.3907 | LR: 1.80e-04 | 1537.7ms/step | 13.0 samples/s
|
| 192 |
+
Step 280/44000 | Loss: 6.3352 | LR: 1.87e-04 | 1538.2ms/step | 13.0 samples/s
|
| 193 |
+
Step 290/44000 | Loss: 6.2995 | LR: 1.93e-04 | 1542.2ms/step | 13.0 samples/s
|
| 194 |
+
Step 300/44000 | Loss: 6.2318 | LR: 2.00e-04 | 1537.5ms/step | 13.0 samples/s
|
| 195 |
+
Step 310/44000 | Loss: 6.1789 | LR: 2.07e-04 | 1540.4ms/step | 13.0 samples/s
|
| 196 |
+
Step 320/44000 | Loss: 6.1509 | LR: 2.13e-04 | 1543.8ms/step | 13.0 samples/s
|
| 197 |
+
Step 330/44000 | Loss: 6.1072 | LR: 2.20e-04 | 1542.9ms/step | 13.0 samples/s
|
| 198 |
+
Step 340/44000 | Loss: 6.0580 | LR: 2.27e-04 | 1540.1ms/step | 13.0 samples/s
|
| 199 |
+
Step 350/44000 | Loss: 6.0190 | LR: 2.33e-04 | 1543.3ms/step | 13.0 samples/s
|
| 200 |
+
Step 360/44000 | Loss: 6.0037 | LR: 2.40e-04 | 1551.3ms/step | 12.9 samples/s
|
| 201 |
+
Step 370/44000 | Loss: 5.9524 | LR: 2.47e-04 | 1543.2ms/step | 13.0 samples/s
|
| 202 |
+
Step 380/44000 | Loss: 5.9209 | LR: 2.53e-04 | 1542.8ms/step | 13.0 samples/s
|
| 203 |
+
Step 390/44000 | Loss: 5.9120 | LR: 2.60e-04 | 1550.1ms/step | 12.9 samples/s
|
| 204 |
+
Step 400/44000 | Loss: 5.8717 | LR: 2.67e-04 | 1546.1ms/step | 12.9 samples/s
|
| 205 |
+
Step 410/44000 | Loss: 5.8164 | LR: 2.73e-04 | 1543.7ms/step | 13.0 samples/s
|
| 206 |
+
Step 420/44000 | Loss: 5.8121 | LR: 2.80e-04 | 1547.0ms/step | 12.9 samples/s
|
| 207 |
+
Step 430/44000 | Loss: 5.7661 | LR: 2.87e-04 | 1549.2ms/step | 12.9 samples/s
|
| 208 |
+
Step 440/44000 | Loss: 5.7407 | LR: 2.93e-04 | 1542.6ms/step | 13.0 samples/s
|
| 209 |
+
Step 450/44000 | Loss: 5.7036 | LR: 3.00e-04 | 1544.2ms/step | 13.0 samples/s
|
| 210 |
+
Step 460/44000 | Loss: 5.6247 | LR: 3.07e-04 | 1547.9ms/step | 12.9 samples/s
|
| 211 |
+
Step 470/44000 | Loss: 5.5889 | LR: 3.13e-04 | 1543.0ms/step | 13.0 samples/s
|
| 212 |
+
Step 480/44000 | Loss: 5.6134 | LR: 3.20e-04 | 1541.4ms/step | 13.0 samples/s
|
| 213 |
+
Step 490/44000 | Loss: 5.5577 | LR: 3.27e-04 | 1545.3ms/step | 12.9 samples/s
|
| 214 |
+
Step 500/44000 | Loss: 5.5126 | LR: 3.33e-04 | 1545.6ms/step | 12.9 samples/s
|
| 215 |
+
Step 510/44000 | Loss: 5.4746 | LR: 3.40e-04 | 1543.7ms/step | 13.0 samples/s
|
| 216 |
+
Step 520/44000 | Loss: 5.4494 | LR: 3.47e-04 | 1542.7ms/step | 13.0 samples/s
|
| 217 |
+
Step 530/44000 | Loss: 5.4386 | LR: 3.53e-04 | 1543.5ms/step | 13.0 samples/s
|
| 218 |
+
Step 540/44000 | Loss: 5.3931 | LR: 3.60e-04 | 1541.1ms/step | 13.0 samples/s
|
| 219 |
+
Step 550/44000 | Loss: 5.3387 | LR: 3.67e-04 | 1538.9ms/step | 13.0 samples/s
|
| 220 |
+
Step 560/44000 | Loss: 5.3424 | LR: 3.73e-04 | 1543.6ms/step | 13.0 samples/s
|
| 221 |
+
Step 570/44000 | Loss: 5.2629 | LR: 3.80e-04 | 1541.5ms/step | 13.0 samples/s
|
| 222 |
+
Step 580/44000 | Loss: 5.2687 | LR: 3.87e-04 | 1540.3ms/step | 13.0 samples/s
|
| 223 |
+
Step 590/44000 | Loss: 5.2660 | LR: 3.93e-04 | 1542.5ms/step | 13.0 samples/s
|
| 224 |
+
Step 600/44000 | Loss: 5.2257 | LR: 4.00e-04 | 1545.8ms/step | 12.9 samples/s
|
| 225 |
+
Step 610/44000 | Loss: 5.2140 | LR: 4.07e-04 | 1540.9ms/step | 13.0 samples/s
|
| 226 |
+
Step 620/44000 | Loss: 5.1616 | LR: 4.13e-04 | 1542.9ms/step | 13.0 samples/s
|
| 227 |
+
Step 630/44000 | Loss: 5.1341 | LR: 4.20e-04 | 1544.6ms/step | 12.9 samples/s
|
| 228 |
+
Step 640/44000 | Loss: 5.1155 | LR: 4.27e-04 | 1543.8ms/step | 13.0 samples/s
|
| 229 |
+
Step 650/44000 | Loss: 5.1029 | LR: 4.33e-04 | 1540.7ms/step | 13.0 samples/s
|
| 230 |
+
Step 660/44000 | Loss: 5.0698 | LR: 4.40e-04 | 1544.1ms/step | 13.0 samples/s
|
| 231 |
+
Step 670/44000 | Loss: 5.0611 | LR: 4.47e-04 | 1544.2ms/step | 13.0 samples/s
|
| 232 |
+
Step 680/44000 | Loss: 5.0476 | LR: 4.53e-04 | 1543.4ms/step | 13.0 samples/s
|
| 233 |
+
Step 690/44000 | Loss: 4.9814 | LR: 4.60e-04 | 1542.5ms/step | 13.0 samples/s
|
| 234 |
+
Step 700/44000 | Loss: 5.0014 | LR: 4.67e-04 | 1545.1ms/step | 12.9 samples/s
|
| 235 |
+
Step 710/44000 | Loss: 4.9079 | LR: 4.73e-04 | 1541.0ms/step | 13.0 samples/s
|
| 236 |
+
Step 720/44000 | Loss: 4.8878 | LR: 4.80e-04 | 1539.4ms/step | 13.0 samples/s
|
| 237 |
+
Step 730/44000 | Loss: 4.9246 | LR: 4.87e-04 | 1543.4ms/step | 13.0 samples/s
|
| 238 |
+
Step 740/44000 | Loss: 4.8546 | LR: 4.93e-04 | 1541.3ms/step | 13.0 samples/s
|
| 239 |
+
Step 750/44000 | Loss: 4.8082 | LR: 5.00e-04 | 1538.1ms/step | 13.0 samples/s
|
| 240 |
+
Step 760/44000 | Loss: 4.7863 | LR: 5.00e-04 | 1536.6ms/step | 13.0 samples/s
|
| 241 |
+
Step 770/44000 | Loss: 4.7749 | LR: 5.00e-04 | 1539.1ms/step | 13.0 samples/s
|
| 242 |
+
Step 780/44000 | Loss: 4.7201 | LR: 5.00e-04 | 1537.7ms/step | 13.0 samples/s
|
| 243 |
+
Step 790/44000 | Loss: 4.7256 | LR: 5.00e-04 | 1536.5ms/step | 13.0 samples/s
|
| 244 |
+
Step 800/44000 | Loss: 4.7247 | LR: 5.00e-04 | 1539.4ms/step | 13.0 samples/s
|
| 245 |
+
Step 810/44000 | Loss: 4.6992 | LR: 5.00e-04 | 1540.2ms/step | 13.0 samples/s
|
| 246 |
+
Step 820/44000 | Loss: 4.6171 | LR: 5.00e-04 | 1541.9ms/step | 13.0 samples/s
|
| 247 |
+
Step 830/44000 | Loss: 4.6268 | LR: 5.00e-04 | 1541.9ms/step | 13.0 samples/s
|
| 248 |
+
Step 840/44000 | Loss: 4.6340 | LR: 5.00e-04 | 1539.6ms/step | 13.0 samples/s
|
| 249 |
+
Step 850/44000 | Loss: 4.5301 | LR: 5.00e-04 | 1536.5ms/step | 13.0 samples/s
|
| 250 |
+
Step 860/44000 | Loss: 4.5799 | LR: 5.00e-04 | 1539.8ms/step | 13.0 samples/s
|
| 251 |
+
Step 870/44000 | Loss: 4.5152 | LR: 5.00e-04 | 1542.2ms/step | 13.0 samples/s
|
| 252 |
+
Step 880/44000 | Loss: 4.5101 | LR: 5.00e-04 | 1540.7ms/step | 13.0 samples/s
|
| 253 |
+
Step 890/44000 | Loss: 4.5257 | LR: 5.00e-04 | 1539.0ms/step | 13.0 samples/s
|
| 254 |
+
Step 900/44000 | Loss: 4.4563 | LR: 5.00e-04 | 1538.1ms/step | 13.0 samples/s
|
| 255 |
+
Step 910/44000 | Loss: 4.4617 | LR: 5.00e-04 | 1540.6ms/step | 13.0 samples/s
|
| 256 |
+
Step 920/44000 | Loss: 4.4013 | LR: 5.00e-04 | 1541.3ms/step | 13.0 samples/s
|
| 257 |
+
Step 930/44000 | Loss: 4.4441 | LR: 5.00e-04 | 1539.9ms/step | 13.0 samples/s
|
| 258 |
+
Step 940/44000 | Loss: 4.3786 | LR: 5.00e-04 | 1539.1ms/step | 13.0 samples/s
|
| 259 |
+
Step 950/44000 | Loss: 4.3946 | LR: 5.00e-04 | 1542.1ms/step | 13.0 samples/s
|
| 260 |
+
Step 960/44000 | Loss: 4.3700 | LR: 5.00e-04 | 1539.1ms/step | 13.0 samples/s
|
| 261 |
+
Step 970/44000 | Loss: 4.3659 | LR: 5.00e-04 | 1538.7ms/step | 13.0 samples/s
|
| 262 |
+
Step 980/44000 | Loss: 4.3307 | LR: 5.00e-04 | 1539.3ms/step | 13.0 samples/s
|
| 263 |
+
Step 990/44000 | Loss: 4.3159 | LR: 5.00e-04 | 1538.2ms/step | 13.0 samples/s
|
| 264 |
+
Step 1000/44000 | Loss: 4.2865 | LR: 5.00e-04 | 1540.0ms/step | 13.0 samples/s
|
epfl_final/train_logs/eval_seed71_dense_50462_quick.log
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[16:13:06] total jobs=22 already-done=0 gpus=[0, 1, 2, 3, 4, 5] n=20
|
| 2 |
+
[16:13:06] START gpu0 passkey_8k_dense_seed71 try1 (~22 remaining)
|
| 3 |
+
[16:13:06] START gpu1 ppl_8k_dense_seed71_resampled_fineweb try1 (~22 remaining)
|
| 4 |
+
[16:13:06] START gpu2 ppl_8k_dense_seed71_resampled_pg19 try1 (~22 remaining)
|
| 5 |
+
[16:13:06] START gpu3 ppl_8k_dense_seed71_resampled_arxiv try1 (~22 remaining)
|
| 6 |
+
[16:13:06] START gpu4 ppl_8k_dense_seed71_fixed_fineweb try1 (~22 remaining)
|
| 7 |
+
[16:13:06] START gpu5 ppl_8k_dense_seed71_fixed_pg19 try1 (~22 remaining)
|
| 8 |
+
[16:24:21] DONE gpu3 ppl_8k_dense_seed71_resampled_arxiv rc=0 ok
|
| 9 |
+
[16:24:21] DONE gpu4 ppl_8k_dense_seed71_fixed_fineweb rc=0 ok
|
| 10 |
+
[16:24:21] DONE gpu5 ppl_8k_dense_seed71_fixed_pg19 rc=0 ok
|
| 11 |
+
[16:24:21] START gpu3 ppl_8k_dense_seed71_fixed_arxiv try1 (~19 remaining)
|
| 12 |
+
[16:24:22] START gpu4 ruler_8k_niah_multikey_dense_seed71 try1 (~19 remaining)
|
| 13 |
+
[16:24:22] START gpu5 ruler_8k_niah_multivalue_dense_seed71 try1 (~19 remaining)
|
| 14 |
+
[16:24:27] DONE gpu4 ruler_8k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 15 |
+
[16:24:27] DONE gpu5 ruler_8k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 16 |
+
[16:24:27] START gpu4 ruler_8k_niah_multikey_dense_seed71 try2 (~18 remaining)
|
| 17 |
+
[16:24:27] START gpu5 ruler_8k_niah_multivalue_dense_seed71 try2 (~18 remaining)
|
| 18 |
+
[16:24:32] DONE gpu1 ppl_8k_dense_seed71_resampled_fineweb rc=0 ok
|
| 19 |
+
[16:24:32] DONE gpu2 ppl_8k_dense_seed71_resampled_pg19 rc=0 ok
|
| 20 |
+
[16:24:32] DONE gpu4 ruler_8k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 21 |
+
[16:24:32] DONE gpu5 ruler_8k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 22 |
+
[16:24:32] START gpu1 ruler_8k_niah_multikey_dense_seed71 try3 (~16 remaining)
|
| 23 |
+
[16:24:32] START gpu2 ruler_8k_niah_multivalue_dense_seed71 try3 (~15 remaining)
|
| 24 |
+
[16:24:32] START gpu4 ruler_8k_vt_dense_seed71 try1 (~15 remaining)
|
| 25 |
+
[16:24:32] START gpu5 kv_h2o_8k_dense_seed71 try1 (~15 remaining)
|
| 26 |
+
[16:24:37] DONE gpu0 passkey_8k_dense_seed71 rc=0 ok
|
| 27 |
+
[16:24:37] DONE gpu1 ruler_8k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 28 |
+
[16:24:37] DONE gpu2 ruler_8k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 29 |
+
[16:24:37] DONE gpu4 ruler_8k_vt_dense_seed71 rc=2 NO-TSV
|
| 30 |
+
[16:24:37] START gpu0 ruler_8k_vt_dense_seed71 try2 (~14 remaining)
|
| 31 |
+
[16:24:37] START gpu1 kv_stream_8k_dense_seed71 try1 (~14 remaining)
|
| 32 |
+
[16:24:38] START gpu2 passkey_30k_dense_seed71 try1 (~14 remaining)
|
| 33 |
+
[16:24:38] START gpu4 ppl_30k_dense_seed71_resampled_pg19 try1 (~14 remaining)
|
| 34 |
+
[16:24:43] DONE gpu0 ruler_8k_vt_dense_seed71 rc=2 NO-TSV
|
| 35 |
+
[16:24:43] START gpu0 ruler_8k_vt_dense_seed71 try3 (~13 remaining)
|
| 36 |
+
[16:24:48] DONE gpu0 ruler_8k_vt_dense_seed71 rc=2 NO-TSV
|
| 37 |
+
[16:24:48] START gpu0 ppl_30k_dense_seed71_resampled_arxiv try1 (~13 remaining)
|
| 38 |
+
[16:26:13] DONE gpu1 kv_stream_8k_dense_seed71 rc=0 ok
|
| 39 |
+
[16:26:13] START gpu1 ppl_30k_dense_seed71_fixed_pg19 try1 (~12 remaining)
|
| 40 |
+
[16:27:48] DONE gpu5 kv_h2o_8k_dense_seed71 rc=0 ok
|
| 41 |
+
[16:27:48] START gpu5 ppl_30k_dense_seed71_fixed_arxiv try1 (~11 remaining)
|
| 42 |
+
[16:34:58] DONE gpu3 ppl_8k_dense_seed71_fixed_arxiv rc=0 ok
|
| 43 |
+
[16:34:58] START gpu3 ruler_30k_niah_multikey_dense_seed71 try1 (~10 remaining)
|
| 44 |
+
[16:35:03] DONE gpu3 ruler_30k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 45 |
+
[16:35:03] START gpu3 ruler_30k_niah_multikey_dense_seed71 try2 (~10 remaining)
|
| 46 |
+
[16:35:08] DONE gpu3 ruler_30k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 47 |
+
[16:35:08] START gpu3 ruler_30k_niah_multikey_dense_seed71 try3 (~9 remaining)
|
| 48 |
+
[16:35:13] DONE gpu3 ruler_30k_niah_multikey_dense_seed71 rc=2 NO-TSV
|
| 49 |
+
[16:35:14] START gpu3 ruler_30k_niah_multivalue_dense_seed71 try1 (~9 remaining)
|
| 50 |
+
[16:35:19] DONE gpu3 ruler_30k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 51 |
+
[16:35:19] START gpu3 ruler_30k_niah_multivalue_dense_seed71 try2 (~9 remaining)
|
| 52 |
+
[16:35:24] DONE gpu3 ruler_30k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 53 |
+
[16:35:24] START gpu3 ruler_30k_niah_multivalue_dense_seed71 try3 (~8 remaining)
|
| 54 |
+
[16:35:29] DONE gpu3 ruler_30k_niah_multivalue_dense_seed71 rc=2 NO-TSV
|
| 55 |
+
[16:35:29] START gpu3 ruler_30k_vt_dense_seed71 try1 (~8 remaining)
|
| 56 |
+
[16:35:34] DONE gpu3 ruler_30k_vt_dense_seed71 rc=2 NO-TSV
|
| 57 |
+
[16:35:34] START gpu3 ruler_30k_vt_dense_seed71 try2 (~8 remaining)
|
| 58 |
+
[16:35:39] DONE gpu3 ruler_30k_vt_dense_seed71 rc=2 NO-TSV
|
| 59 |
+
[16:35:39] START gpu3 ruler_30k_vt_dense_seed71 try3 (~7 remaining)
|
| 60 |
+
[16:35:44] DONE gpu3 ruler_30k_vt_dense_seed71 rc=2 NO-TSV
|
| 61 |
+
[16:35:44] START gpu3 kv_h2o_30k_dense_seed71 try1 (~7 remaining)
|
| 62 |
+
[16:39:09] DONE gpu3 kv_h2o_30k_dense_seed71 rc=0 ok
|
| 63 |
+
[16:39:09] START gpu3 kv_stream_30k_dense_seed71 try1 (~6 remaining)
|
| 64 |
+
[16:40:39] DONE gpu3 kv_stream_30k_dense_seed71 rc=0 ok
|
epfl_final/train_logs/launch_seed72_mixed_after_dense70.log
ADDED
|
File without changes
|
epfl_final/train_logs/launch_seed72_mixed_after_dense70.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3129
|
epfl_final/train_logs/launch_seed72_mixed_after_dense70.sh
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -u
|
| 3 |
+
while ps -eo args= | grep -F -- "train.py --config configs/p2_1p2b_dense_seed70_6gpu_resume41750.json" | grep -v grep >/dev/null; do sleep 30; done
|
| 4 |
+
cd /mloscratch/homes/navasard/sparse/SparseAtt
|
| 5 |
+
log=/mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.log
|
| 6 |
+
if ps -eo args= | grep -F -- "train.py --config configs/p2_1p2b_mixed_seed72_6gpu_after_dense70.json" | grep -v grep >/dev/null; then exit 0; fi
|
| 7 |
+
nohup /mloscratch/homes/navasard/conda/bin/torchrun --standalone --nproc_per_node=6 train.py --config configs/p2_1p2b_mixed_seed72_6gpu_after_dense70.json >"$log" 2>&1 < /dev/null &
|
| 8 |
+
echo $! > /mloscratch/homes/navasard/sparse/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.pid
|
epfl_final/train_logs/p0_frozen_matched_20260821.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
120 jobs pending (120 total)
|
epfl_final/train_logs/p0_frozen_matched_20260821.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
9495
|
epfl_final/train_logs/p0_frozen_matched_20260821_resume.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
8 jobs pending (120 total)
|
epfl_final/train_logs/p0_frozen_matched_20260821_resume.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
127
|
epfl_final/train_logs/p0_frozen_matched_a100_20260821.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
8 jobs pending (120 total)
|
epfl_final/train_logs/p0_frozen_matched_a100_20260821.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
205
|
epfl_final/train_logs/p1_control_eval_scheduler_20260822.log
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Sat Aug 22 01:04:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 2 |
+
Sat Aug 22 01:04:28 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 3 |
+
Sat Aug 22 01:05:08 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 4 |
+
Sat Aug 22 01:06:08 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 5 |
+
Sat Aug 22 01:07:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 6 |
+
Sat Aug 22 01:08:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 7 |
+
Sat Aug 22 01:09:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 8 |
+
Sat Aug 22 01:10:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 9 |
+
Sat Aug 22 01:11:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 10 |
+
Sat Aug 22 01:12:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 11 |
+
Sat Aug 22 01:13:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 12 |
+
Sat Aug 22 01:14:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 13 |
+
Sat Aug 22 01:15:09 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 14 |
+
Sat Aug 22 01:16:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 15 |
+
Sat Aug 22 01:17:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 16 |
+
Sat Aug 22 01:18:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 17 |
+
Sat Aug 22 01:19:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 18 |
+
Sat Aug 22 01:20:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 19 |
+
Sat Aug 22 01:21:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 20 |
+
Sat Aug 22 01:22:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 21 |
+
Sat Aug 22 01:23:10 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 22 |
+
Sat Aug 22 01:24:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 23 |
+
Sat Aug 22 01:25:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 24 |
+
Sat Aug 22 01:26:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 25 |
+
Sat Aug 22 01:27:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 26 |
+
Sat Aug 22 01:28:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 27 |
+
Sat Aug 22 01:29:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 28 |
+
Sat Aug 22 01:30:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 29 |
+
Sat Aug 22 01:31:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 30 |
+
Sat Aug 22 01:32:11 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 31 |
+
Sat Aug 22 01:33:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 32 |
+
Sat Aug 22 01:34:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 33 |
+
Sat Aug 22 01:35:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 34 |
+
Sat Aug 22 01:36:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 35 |
+
Sat Aug 22 01:37:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 36 |
+
Sat Aug 22 01:38:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 37 |
+
Sat Aug 22 01:39:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 38 |
+
Sat Aug 22 01:40:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 39 |
+
Sat Aug 22 01:41:12 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 40 |
+
Sat Aug 22 01:42:13 PM UTC 2026 checkpoints=0/6 active_static46=86 busy_gpus=5
|
| 41 |
+
Sat Aug 22 01:43:13 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 42 |
+
Sat Aug 22 01:44:13 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 43 |
+
Sat Aug 22 01:45:13 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 44 |
+
Sat Aug 22 01:46:13 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 45 |
+
Sat Aug 22 01:47:13 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 46 |
+
Sat Aug 22 01:48:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 47 |
+
Sat Aug 22 01:49:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 48 |
+
Sat Aug 22 01:50:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 49 |
+
Sat Aug 22 01:51:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 50 |
+
Sat Aug 22 01:52:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 51 |
+
Sat Aug 22 01:53:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 52 |
+
Sat Aug 22 01:54:14 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 53 |
+
Sat Aug 22 01:55:15 PM UTC 2026 checkpoints=0/6 active_static46=0 busy_gpus=0
|
| 54 |
+
Sat Aug 22 01:55:41 PM UTC 2026 checkpoints=6/6 active_static46=0 busy_gpus=0
|
| 55 |
+
jobs=72 pending=72
|
| 56 |
+
Traceback (most recent call last):
|
| 57 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 30, in <module>
|
| 58 |
+
if __name__=="__main__":main()
|
| 59 |
+
~~~~^^
|
| 60 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 28, in main
|
| 61 |
+
for f in [ex.submit(run,(i,j)) for i,j in enumerate(pend)]:f.result()
|
| 62 |
+
~~~~~~~~^^
|
| 63 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/_base.py", line 456, in result
|
| 64 |
+
return self.__get_result()
|
| 65 |
+
~~~~~~~~~~~~~~~~~^^
|
| 66 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/_base.py", line 401, in __get_result
|
| 67 |
+
raise self._exception
|
| 68 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/thread.py", line 59, in run
|
| 69 |
+
result = self.fn(*self.args, **self.kwargs)
|
| 70 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 26, in run
|
| 71 |
+
if rc or not complete(p):raise RuntimeError(f"{tag} {L} {co} rc={rc}")
|
| 72 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 73 |
+
RuntimeError: static42 8192 arxiv rc=1
|
| 74 |
+
Sat Aug 22 01:58:52 PM UTC 2026 checkpoints=6/6 active_static46=0 busy_gpus=0
|
| 75 |
+
jobs=72 pending=72
|
| 76 |
+
Traceback (most recent call last):
|
| 77 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 30, in <module>
|
| 78 |
+
if __name__=="__main__":main()
|
| 79 |
+
~~~~^^
|
| 80 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 28, in main
|
| 81 |
+
for f in [ex.submit(run,(i,j)) for i,j in enumerate(pend)]:f.result()
|
| 82 |
+
~~~~~~~~^^
|
| 83 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/_base.py", line 456, in result
|
| 84 |
+
return self.__get_result()
|
| 85 |
+
~~~~~~~~~~~~~~~~~^^
|
| 86 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/_base.py", line 401, in __get_result
|
| 87 |
+
raise self._exception
|
| 88 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/concurrent/futures/thread.py", line 59, in run
|
| 89 |
+
result = self.fn(*self.args, **self.kwargs)
|
| 90 |
+
File "/mloscratch/homes/navasard/sparse/SparseAtt/eval/run_p1_control_matched.py", line 26, in run
|
| 91 |
+
if rc or not complete(p):raise RuntimeError(f"{tag} {L} {co} rc={rc}")
|
| 92 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 93 |
+
RuntimeError: static42 8192 arxiv rc=1
|
| 94 |
+
Sat Aug 22 02:12:32 PM UTC 2026 checkpoints=6/6 active_static46=0 busy_gpus=0
|
| 95 |
+
jobs=72 pending=72
|
| 96 |
+
all matched control evaluations complete
|
| 97 |
+
jobs=96 pending=24
|
| 98 |
+
all matched control evaluations complete
|
epfl_final/train_logs/p1_control_eval_scheduler_20260823.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
jobs=144 pending=48
|
| 2 |
+
all matched control evaluations complete
|
epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu.log
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 2,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 2gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500
|
| 69 |
+
Checkpoint activations: FalseCheckpoint activations: False
|
| 70 |
+
|
| 71 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
================================================================================
|
| 75 |
+
MODEL ARCHITECTURE
|
| 76 |
+
================================================================================
|
| 77 |
+
Model config:
|
| 78 |
+
vocab_size: 151936
|
| 79 |
+
hidden_size: 1536
|
| 80 |
+
num_hidden_layers: 28
|
| 81 |
+
num_attention_heads: 12
|
| 82 |
+
num_key_value_heads: 4
|
| 83 |
+
head_dim: 128
|
| 84 |
+
intermediate_size: 6144
|
| 85 |
+
max_position_embeddings: 32768
|
| 86 |
+
rope_theta: 1000000
|
| 87 |
+
rope_train_position_scale_set: None
|
| 88 |
+
attention_type: sparse_stochastic
|
| 89 |
+
sparse_block_size: 128
|
| 90 |
+
sparse_num_random_blocks: 31
|
| 91 |
+
sparse_mask_pool_size: 1
|
| 92 |
+
sparse_max_seq_length: 8192
|
| 93 |
+
|
| 94 |
+
Parameter count:
|
| 95 |
+
Total parameters: 1,202,352,640
|
| 96 |
+
Trainable parameters: 1,202,352,640
|
| 97 |
+
Non-trainable parameters: 0
|
| 98 |
+
Model size (BF16): 2.40 GB
|
| 99 |
+
|
| 100 |
+
--------------------------------------------------------------------------------
|
| 101 |
+
LAYER-BY-LAYER SUMMARY
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
================================================================================================================================================================
|
| 104 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 107 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True
|
| 108 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True
|
| 109 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True
|
| 139 |
+
├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True
|
| 140 |
+
================================================================================================================================================================
|
| 141 |
+
Total params: 1,435,726,336
|
| 142 |
+
Trainable params: 1,435,726,336
|
| 143 |
+
Non-trainable params: 0
|
| 144 |
+
Total mult-adds (Units.GIGABYTES): 2.87
|
| 145 |
+
================================================================================================================================================================
|
| 146 |
+
Input size (MB): 0.13
|
| 147 |
+
Forward/backward pass size (MB): 52437.19
|
| 148 |
+
Params size (MB): 2871.45
|
| 149 |
+
Estimated Total Size (MB): 55308.77
|
| 150 |
+
================================================================================================================================================================
|
| 151 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 152 |
+
================================================================================
|
| 153 |
+
|
| 154 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 155 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 156 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
|
| 158 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 159 |
+
|
| 160 |
+
Dataset split: 3027764 train, 159356 validation
|
| 161 |
+
Training examples (packed, train split): 3027764
|
| 162 |
+
Validation examples (packed, val split): 159356
|
| 163 |
+
Per-device batch size: 2
|
| 164 |
+
World size (GPUs): 2
|
| 165 |
+
Gradient accumulation steps: 15
|
| 166 |
+
Effective batch size (seqs/step): 60
|
| 167 |
+
Sequence length: 8192
|
| 168 |
+
Tokens per optimizer step: 491,520
|
| 169 |
+
Micro-batches per epoch (per rank): 756941
|
| 170 |
+
Optimizer steps per epoch: 50462
|
| 171 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 172 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500...
|
| 173 |
+
Preserving checkpoint LR state for exact resume: current=[0.0004231431036937715], schedule_base=[0.0005]
|
| 174 |
+
Resumed from checkpoint at step 13500 (epoch 0, micro_step 176244)
|
| 175 |
+
Resuming training from step 13500
|
| 176 |
+
Resuming wandb run rgmkfpnl
|
| 177 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 178 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 179 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 180 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_132232-rgmkfpnl
|
| 181 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 182 |
+
wandb: Resuming run 1.2b dense 8k seed70 2gpu resume
|
| 183 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 184 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 185 |
+
Starting training...
|
| 186 |
+
Validation at step 13500: Loss = 2.6953, Perplexity = 14.81
|
| 187 |
+
wandb: WARNING Tried to log to step 13500 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 188 |
+
Switching attention mode for step 13501: dense (dense step interval)
|
| 189 |
+
Step 13510/50462 | Loss: 2.4722 | LR: 4.23e-04 | 54319.4ms/step | 1.1 samples/s
|
| 190 |
+
wandb: WARNING Tried to log to step 13510 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 191 |
+
Step 13520/50462 | Loss: 2.6271 | LR: 4.23e-04 | 5264.0ms/step | 11.4 samples/s
|
| 192 |
+
wandb: WARNING Tried to log to step 13520 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 193 |
+
Step 13530/50462 | Loss: 2.6168 | LR: 4.23e-04 | 5274.7ms/step | 11.4 samples/s
|
| 194 |
+
wandb: WARNING Tried to log to step 13530 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 195 |
+
Step 13540/50462 | Loss: 2.6109 | LR: 4.23e-04 | 5270.0ms/step | 11.4 samples/s
|
| 196 |
+
wandb: WARNING Tried to log to step 13540 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 197 |
+
Step 13550/50462 | Loss: 2.6349 | LR: 4.23e-04 | 5272.2ms/step | 11.4 samples/s
|
| 198 |
+
wandb: WARNING Tried to log to step 13550 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 199 |
+
Step 13560/50462 | Loss: 2.6069 | LR: 4.22e-04 | 5280.6ms/step | 11.4 samples/s
|
| 200 |
+
wandb: WARNING Tried to log to step 13560 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 201 |
+
Step 13570/50462 | Loss: 2.5938 | LR: 4.22e-04 | 5275.3ms/step | 11.4 samples/s
|
| 202 |
+
wandb: WARNING Tried to log to step 13570 that is less than the current step 13571. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 203 |
+
Step 13580/50462 | Loss: 2.6238 | LR: 4.22e-04 | 5278.3ms/step | 11.4 samples/s
|
| 204 |
+
Step 13590/50462 | Loss: 2.6244 | LR: 4.22e-04 | 5276.8ms/step | 11.4 samples/s
|
| 205 |
+
Step 13600/50462 | Loss: 2.6255 | LR: 4.22e-04 | 5281.6ms/step | 11.4 samples/s
|
| 206 |
+
Step 13610/50462 | Loss: 2.6189 | LR: 4.22e-04 | 5277.3ms/step | 11.4 samples/s
|
| 207 |
+
Step 13620/50462 | Loss: 2.6123 | LR: 4.22e-04 | 5267.3ms/step | 11.4 samples/s
|
| 208 |
+
Step 13630/50462 | Loss: 2.5956 | LR: 4.22e-04 | 5280.2ms/step | 11.4 samples/s
|
| 209 |
+
Step 13640/50462 | Loss: 2.6044 | LR: 4.22e-04 | 5284.6ms/step | 11.4 samples/s
|
| 210 |
+
Step 13650/50462 | Loss: 2.5911 | LR: 4.21e-04 | 5277.2ms/step | 11.4 samples/s
|
| 211 |
+
Step 13660/50462 | Loss: 2.5980 | LR: 4.21e-04 | 5282.2ms/step | 11.4 samples/s
|
| 212 |
+
Step 13670/50462 | Loss: 2.6001 | LR: 4.21e-04 | 5274.3ms/step | 11.4 samples/s
|
| 213 |
+
Step 13680/50462 | Loss: 2.5996 | LR: 4.21e-04 | 5276.5ms/step | 11.4 samples/s
|
| 214 |
+
Step 13690/50462 | Loss: 2.5965 | LR: 4.21e-04 | 5270.3ms/step | 11.4 samples/s
|
| 215 |
+
Step 13700/50462 | Loss: 2.6122 | LR: 4.21e-04 | 5272.4ms/step | 11.4 samples/s
|
| 216 |
+
Step 13710/50462 | Loss: 2.5976 | LR: 4.21e-04 | 5281.6ms/step | 11.4 samples/s
|
| 217 |
+
Step 13720/50462 | Loss: 2.5974 | LR: 4.21e-04 | 5274.9ms/step | 11.4 samples/s
|
| 218 |
+
Step 13730/50462 | Loss: 2.6090 | LR: 4.21e-04 | 5269.0ms/step | 11.4 samples/s
|
| 219 |
+
Step 13740/50462 | Loss: 2.6212 | LR: 4.20e-04 | 5271.2ms/step | 11.4 samples/s
|
| 220 |
+
Step 13750/50462 | Loss: 2.6233 | LR: 4.20e-04 | 5279.3ms/step | 11.4 samples/s
|
| 221 |
+
Validation at step 13750: Loss = 2.6969, Perplexity = 14.83
|
| 222 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13750
|
| 223 |
+
Step 13760/50462 | Loss: 2.6068 | LR: 4.20e-04 | 10192.1ms/step | 5.9 samples/s
|
| 224 |
+
Step 13770/50462 | Loss: 2.6057 | LR: 4.20e-04 | 5285.9ms/step | 11.4 samples/s
|
| 225 |
+
Step 13780/50462 | Loss: 2.6062 | LR: 4.20e-04 | 5290.7ms/step | 11.3 samples/s
|
| 226 |
+
Step 13790/50462 | Loss: 2.6155 | LR: 4.20e-04 | 5297.9ms/step | 11.3 samples/s
|
| 227 |
+
Step 13800/50462 | Loss: 2.5954 | LR: 4.20e-04 | 5295.6ms/step | 11.3 samples/s
|
| 228 |
+
Step 13810/50462 | Loss: 2.5958 | LR: 4.20e-04 | 5299.0ms/step | 11.3 samples/s
|
| 229 |
+
Step 13820/50462 | Loss: 2.6024 | LR: 4.19e-04 | 5297.2ms/step | 11.3 samples/s
|
| 230 |
+
Step 13830/50462 | Loss: 2.5973 | LR: 4.19e-04 | 5304.4ms/step | 11.3 samples/s
|
| 231 |
+
Step 13840/50462 | Loss: 2.6033 | LR: 4.19e-04 | 5310.4ms/step | 11.3 samples/s
|
| 232 |
+
Step 13850/50462 | Loss: 2.5919 | LR: 4.19e-04 | 5290.7ms/step | 11.3 samples/s
|
| 233 |
+
Step 13860/50462 | Loss: 2.5948 | LR: 4.19e-04 | 5297.0ms/step | 11.3 samples/s
|
| 234 |
+
Step 13870/50462 | Loss: 2.6241 | LR: 4.19e-04 | 5296.4ms/step | 11.3 samples/s
|
| 235 |
+
Step 13880/50462 | Loss: 2.6233 | LR: 4.19e-04 | 5300.5ms/step | 11.3 samples/s
|
| 236 |
+
Step 13890/50462 | Loss: 2.6220 | LR: 4.19e-04 | 5300.9ms/step | 11.3 samples/s
|
| 237 |
+
Step 13900/50462 | Loss: 2.6202 | LR: 4.19e-04 | 5300.6ms/step | 11.3 samples/s
|
| 238 |
+
Step 13910/50462 | Loss: 2.6004 | LR: 4.18e-04 | 5298.3ms/step | 11.3 samples/s
|
| 239 |
+
Step 13920/50462 | Loss: 2.6166 | LR: 4.18e-04 | 5294.9ms/step | 11.3 samples/s
|
| 240 |
+
Step 13930/50462 | Loss: 2.5898 | LR: 4.18e-04 | 5292.7ms/step | 11.3 samples/s
|
| 241 |
+
Step 13940/50462 | Loss: 2.6027 | LR: 4.18e-04 | 5284.0ms/step | 11.4 samples/s
|
| 242 |
+
Step 13950/50462 | Loss: 2.5709 | LR: 4.18e-04 | 5293.2ms/step | 11.3 samples/s
|
| 243 |
+
Step 13960/50462 | Loss: 2.5982 | LR: 4.18e-04 | 5287.4ms/step | 11.3 samples/s
|
| 244 |
+
Step 13970/50462 | Loss: 2.6064 | LR: 4.18e-04 | 5290.6ms/step | 11.3 samples/s
|
| 245 |
+
Step 13980/50462 | Loss: 2.5810 | LR: 4.18e-04 | 5298.9ms/step | 11.3 samples/s
|
| 246 |
+
Step 13990/50462 | Loss: 2.6062 | LR: 4.17e-04 | 5306.8ms/step | 11.3 samples/s
|
| 247 |
+
Step 14000/50462 | Loss: 2.5894 | LR: 4.17e-04 | 5306.1ms/step | 11.3 samples/s
|
| 248 |
+
Validation at step 14000: Loss = 2.6971, Perplexity = 14.84
|
| 249 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14000
|
| 250 |
+
Step 14010/50462 | Loss: 2.6314 | LR: 4.17e-04 | 10219.2ms/step | 5.9 samples/s
|
| 251 |
+
Step 14020/50462 | Loss: 2.5881 | LR: 4.17e-04 | 5289.3ms/step | 11.3 samples/s
|
| 252 |
+
Step 14030/50462 | Loss: 2.6019 | LR: 4.17e-04 | 5296.4ms/step | 11.3 samples/s
|
| 253 |
+
Step 14040/50462 | Loss: 2.6026 | LR: 4.17e-04 | 5298.8ms/step | 11.3 samples/s
|
| 254 |
+
Step 14050/50462 | Loss: 2.6193 | LR: 4.17e-04 | 5294.1ms/step | 11.3 samples/s
|
| 255 |
+
Step 14060/50462 | Loss: 2.6171 | LR: 4.17e-04 | 5295.9ms/step | 11.3 samples/s
|
| 256 |
+
Step 14070/50462 | Loss: 2.5912 | LR: 4.17e-04 | 5299.9ms/step | 11.3 samples/s
|
| 257 |
+
Step 14080/50462 | Loss: 2.5981 | LR: 4.16e-04 | 5279.6ms/step | 11.4 samples/s
|
| 258 |
+
Step 14090/50462 | Loss: 2.6275 | LR: 4.16e-04 | 5280.1ms/step | 11.4 samples/s
|
| 259 |
+
Step 14100/50462 | Loss: 2.5862 | LR: 4.16e-04 | 5296.0ms/step | 11.3 samples/s
|
| 260 |
+
Step 14110/50462 | Loss: 2.5917 | LR: 4.16e-04 | 5289.9ms/step | 11.3 samples/s
|
| 261 |
+
Step 14120/50462 | Loss: 2.6106 | LR: 4.16e-04 | 5295.7ms/step | 11.3 samples/s
|
| 262 |
+
Step 14130/50462 | Loss: 2.5950 | LR: 4.16e-04 | 5289.9ms/step | 11.3 samples/s
|
| 263 |
+
Step 14140/50462 | Loss: 2.6024 | LR: 4.16e-04 | 5296.1ms/step | 11.3 samples/s
|
| 264 |
+
Step 14150/50462 | Loss: 2.5973 | LR: 4.16e-04 | 5294.2ms/step | 11.3 samples/s
|
| 265 |
+
Step 14160/50462 | Loss: 2.6174 | LR: 4.15e-04 | 5290.3ms/step | 11.3 samples/s
|
| 266 |
+
Step 14170/50462 | Loss: 2.6277 | LR: 4.15e-04 | 5292.4ms/step | 11.3 samples/s
|
| 267 |
+
Step 14180/50462 | Loss: 2.6080 | LR: 4.15e-04 | 5290.4ms/step | 11.3 samples/s
|
| 268 |
+
Step 14190/50462 | Loss: 2.6026 | LR: 4.15e-04 | 5301.3ms/step | 11.3 samples/s
|
| 269 |
+
Step 14200/50462 | Loss: 2.6050 | LR: 4.15e-04 | 5276.9ms/step | 11.4 samples/s
|
| 270 |
+
Step 14210/50462 | Loss: 2.5928 | LR: 4.15e-04 | 5287.1ms/step | 11.3 samples/s
|
| 271 |
+
Step 14220/50462 | Loss: 2.6176 | LR: 4.15e-04 | 5295.8ms/step | 11.3 samples/s
|
| 272 |
+
Step 14230/50462 | Loss: 2.6042 | LR: 4.15e-04 | 5285.7ms/step | 11.4 samples/s
|
| 273 |
+
Step 14240/50462 | Loss: 2.6180 | LR: 4.15e-04 | 5277.8ms/step | 11.4 samples/s
|
| 274 |
+
Step 14250/50462 | Loss: 2.5955 | LR: 4.14e-04 | 5283.2ms/step | 11.4 samples/s
|
| 275 |
+
Validation at step 14250: Loss = 2.6976, Perplexity = 14.84
|
| 276 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14250
|
| 277 |
+
Step 14260/50462 | Loss: 2.5887 | LR: 4.14e-04 | 10155.1ms/step | 5.9 samples/s
|
| 278 |
+
Step 14270/50462 | Loss: 2.5967 | LR: 4.14e-04 | 5273.3ms/step | 11.4 samples/s
|
| 279 |
+
Step 14280/50462 | Loss: 2.5996 | LR: 4.14e-04 | 5284.9ms/step | 11.4 samples/s
|
| 280 |
+
Step 14290/50462 | Loss: 2.5783 | LR: 4.14e-04 | 5289.6ms/step | 11.3 samples/s
|
| 281 |
+
Step 14300/50462 | Loss: 2.6125 | LR: 4.14e-04 | 5293.4ms/step | 11.3 samples/s
|
| 282 |
+
Step 14310/50462 | Loss: 2.5902 | LR: 4.14e-04 | 5296.2ms/step | 11.3 samples/s
|
| 283 |
+
Step 14320/50462 | Loss: 2.5892 | LR: 4.14e-04 | 5292.2ms/step | 11.3 samples/s
|
| 284 |
+
Step 14330/50462 | Loss: 2.6244 | LR: 4.13e-04 | 5297.1ms/step | 11.3 samples/s
|
| 285 |
+
Step 14340/50462 | Loss: 2.5663 | LR: 4.13e-04 | 5288.7ms/step | 11.3 samples/s
|
| 286 |
+
Step 14350/50462 | Loss: 2.5948 | LR: 4.13e-04 | 5293.3ms/step | 11.3 samples/s
|
| 287 |
+
Step 14360/50462 | Loss: 2.6006 | LR: 4.13e-04 | 5291.7ms/step | 11.3 samples/s
|
| 288 |
+
Step 14370/50462 | Loss: 2.5762 | LR: 4.13e-04 | 5293.3ms/step | 11.3 samples/s
|
| 289 |
+
Step 14380/50462 | Loss: 2.5736 | LR: 4.13e-04 | 5296.3ms/step | 11.3 samples/s
|
| 290 |
+
Step 14390/50462 | Loss: 2.5527 | LR: 4.13e-04 | 5280.8ms/step | 11.4 samples/s
|
| 291 |
+
Step 14400/50462 | Loss: 2.6021 | LR: 4.13e-04 | 5291.8ms/step | 11.3 samples/s
|
| 292 |
+
Step 14410/50462 | Loss: 2.6074 | LR: 4.12e-04 | 5299.8ms/step | 11.3 samples/s
|
| 293 |
+
Step 14420/50462 | Loss: 2.5956 | LR: 4.12e-04 | 5295.0ms/step | 11.3 samples/s
|
| 294 |
+
Step 14430/50462 | Loss: 2.5700 | LR: 4.12e-04 | 5295.4ms/step | 11.3 samples/s
|
| 295 |
+
Step 14440/50462 | Loss: 2.5924 | LR: 4.12e-04 | 5294.8ms/step | 11.3 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_relaunch.log
ADDED
|
@@ -0,0 +1,893 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0824 18:16:20.880000 140 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0824 18:16:20.880000 140 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0824 18:16:20.880000 140 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0824 18:16:20.880000 140 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 2,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 2gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
================================================================================
|
| 75 |
+
MODEL ARCHITECTURE
|
| 76 |
+
================================================================================
|
| 77 |
+
Model config:
|
| 78 |
+
vocab_size: 151936
|
| 79 |
+
hidden_size: 1536
|
| 80 |
+
num_hidden_layers: 28
|
| 81 |
+
num_attention_heads: 12
|
| 82 |
+
num_key_value_heads: 4
|
| 83 |
+
head_dim: 128
|
| 84 |
+
intermediate_size: 6144
|
| 85 |
+
max_position_embeddings: 32768
|
| 86 |
+
rope_theta: 1000000
|
| 87 |
+
rope_train_position_scale_set: None
|
| 88 |
+
attention_type: sparse_stochastic
|
| 89 |
+
sparse_block_size: 128
|
| 90 |
+
sparse_num_random_blocks: 31
|
| 91 |
+
sparse_mask_pool_size: 1
|
| 92 |
+
sparse_max_seq_length: 8192
|
| 93 |
+
|
| 94 |
+
Parameter count:
|
| 95 |
+
Total parameters: 1,202,352,640
|
| 96 |
+
Trainable parameters: 1,202,352,640
|
| 97 |
+
Non-trainable parameters: 0
|
| 98 |
+
Model size (BF16): 2.40 GB
|
| 99 |
+
|
| 100 |
+
--------------------------------------------------------------------------------
|
| 101 |
+
LAYER-BY-LAYER SUMMARY
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
================================================================================================================================================================
|
| 104 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 107 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True
|
| 108 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True
|
| 109 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True
|
| 139 |
+
├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True
|
| 140 |
+
================================================================================================================================================================
|
| 141 |
+
Total params: 1,435,726,336
|
| 142 |
+
Trainable params: 1,435,726,336
|
| 143 |
+
Non-trainable params: 0
|
| 144 |
+
Total mult-adds (Units.GIGABYTES): 2.87
|
| 145 |
+
================================================================================================================================================================
|
| 146 |
+
Input size (MB): 0.13
|
| 147 |
+
Forward/backward pass size (MB): 52437.19
|
| 148 |
+
Params size (MB): 2871.45
|
| 149 |
+
Estimated Total Size (MB): 55308.77
|
| 150 |
+
================================================================================================================================================================
|
| 151 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 152 |
+
================================================================================
|
| 153 |
+
|
| 154 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 155 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 156 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 159 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 160 |
+
Dataset split: 3027764 train, 159356 validation
|
| 161 |
+
Training examples (packed, train split): 3027764
|
| 162 |
+
Validation examples (packed, val split): 159356
|
| 163 |
+
Per-device batch size: 2
|
| 164 |
+
World size (GPUs): 2
|
| 165 |
+
Gradient accumulation steps: 15
|
| 166 |
+
Effective batch size (seqs/step): 60
|
| 167 |
+
Sequence length: 8192
|
| 168 |
+
Tokens per optimizer step: 491,520
|
| 169 |
+
Micro-batches per epoch (per rank): 756941
|
| 170 |
+
Optimizer steps per epoch: 50462
|
| 171 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 172 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500...
|
| 173 |
+
Preserving checkpoint LR state for exact resume: current=[0.0004231431036937715], schedule_base=[0.0005]
|
| 174 |
+
Resumed from checkpoint at step 13500 (epoch 0, micro_step 176244)
|
| 175 |
+
Resuming training from step 13500
|
| 176 |
+
Resuming wandb run rgmkfpnl
|
| 177 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 178 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 179 |
+
wandb: setting up run rgmkfpnl
|
| 180 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 181 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_181749-rgmkfpnl
|
| 182 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 183 |
+
wandb: Resuming run 1.2b dense 8k seed70 2gpu resume
|
| 184 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 185 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 186 |
+
Starting training...
|
| 187 |
+
Validation at step 13500: Loss = 2.6953, Perplexity = 14.81
|
| 188 |
+
wandb: WARNING Tried to log to step 13500 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 189 |
+
Switching attention mode for step 13501: dense (dense step interval)
|
| 190 |
+
Step 13510/50462 | Loss: 2.4722 | LR: 4.23e-04 | 42665.0ms/step | 1.4 samples/s
|
| 191 |
+
wandb: WARNING Tried to log to step 13510 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 192 |
+
Step 13520/50462 | Loss: 2.6271 | LR: 4.23e-04 | 5239.1ms/step | 11.5 samples/s
|
| 193 |
+
wandb: WARNING Tried to log to step 13520 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 194 |
+
Step 13530/50462 | Loss: 2.6167 | LR: 4.23e-04 | 5244.6ms/step | 11.4 samples/s
|
| 195 |
+
wandb: WARNING Tried to log to step 13530 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 196 |
+
Step 13540/50462 | Loss: 2.6109 | LR: 4.23e-04 | 5249.7ms/step | 11.4 samples/s
|
| 197 |
+
wandb: WARNING Tried to log to step 13540 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 198 |
+
Step 13550/50462 | Loss: 2.6348 | LR: 4.23e-04 | 5254.4ms/step | 11.4 samples/s
|
| 199 |
+
wandb: WARNING Tried to log to step 13550 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Step 13560/50462 | Loss: 2.6069 | LR: 4.22e-04 | 5257.1ms/step | 11.4 samples/s
|
| 201 |
+
wandb: WARNING Tried to log to step 13560 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 202 |
+
Step 13570/50462 | Loss: 2.5938 | LR: 4.22e-04 | 5253.4ms/step | 11.4 samples/s
|
| 203 |
+
wandb: WARNING Tried to log to step 13570 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 204 |
+
Step 13580/50462 | Loss: 2.6239 | LR: 4.22e-04 | 5255.6ms/step | 11.4 samples/s
|
| 205 |
+
wandb: WARNING Tried to log to step 13580 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 206 |
+
Step 13590/50462 | Loss: 2.6244 | LR: 4.22e-04 | 5252.6ms/step | 11.4 samples/s
|
| 207 |
+
wandb: WARNING Tried to log to step 13590 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 208 |
+
Step 13600/50462 | Loss: 2.6255 | LR: 4.22e-04 | 5253.2ms/step | 11.4 samples/s
|
| 209 |
+
wandb: WARNING Tried to log to step 13600 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Step 13610/50462 | Loss: 2.6189 | LR: 4.22e-04 | 5252.0ms/step | 11.4 samples/s
|
| 211 |
+
wandb: WARNING Tried to log to step 13610 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 212 |
+
Step 13620/50462 | Loss: 2.6123 | LR: 4.22e-04 | 5254.3ms/step | 11.4 samples/s
|
| 213 |
+
wandb: WARNING Tried to log to step 13620 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Step 13630/50462 | Loss: 2.5956 | LR: 4.22e-04 | 5253.1ms/step | 11.4 samples/s
|
| 215 |
+
wandb: WARNING Tried to log to step 13630 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 216 |
+
Step 13640/50462 | Loss: 2.6044 | LR: 4.22e-04 | 5251.2ms/step | 11.4 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 13640 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 13650/50462 | Loss: 2.5911 | LR: 4.21e-04 | 5244.5ms/step | 11.4 samples/s
|
| 219 |
+
wandb: WARNING Tried to log to step 13650 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 220 |
+
Step 13660/50462 | Loss: 2.5980 | LR: 4.21e-04 | 5253.5ms/step | 11.4 samples/s
|
| 221 |
+
wandb: WARNING Tried to log to step 13660 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 222 |
+
Step 13670/50462 | Loss: 2.6001 | LR: 4.21e-04 | 5254.5ms/step | 11.4 samples/s
|
| 223 |
+
wandb: WARNING Tried to log to step 13670 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 224 |
+
Step 13680/50462 | Loss: 2.5996 | LR: 4.21e-04 | 5255.0ms/step | 11.4 samples/s
|
| 225 |
+
wandb: WARNING Tried to log to step 13680 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 226 |
+
Step 13690/50462 | Loss: 2.5965 | LR: 4.21e-04 | 5256.4ms/step | 11.4 samples/s
|
| 227 |
+
wandb: WARNING Tried to log to step 13690 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 228 |
+
Step 13700/50462 | Loss: 2.6121 | LR: 4.21e-04 | 5259.7ms/step | 11.4 samples/s
|
| 229 |
+
wandb: WARNING Tried to log to step 13700 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 230 |
+
Step 13710/50462 | Loss: 2.5976 | LR: 4.21e-04 | 5248.0ms/step | 11.4 samples/s
|
| 231 |
+
wandb: WARNING Tried to log to step 13710 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 232 |
+
Step 13720/50462 | Loss: 2.5975 | LR: 4.21e-04 | 5255.3ms/step | 11.4 samples/s
|
| 233 |
+
wandb: WARNING Tried to log to step 13720 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 234 |
+
Step 13730/50462 | Loss: 2.6090 | LR: 4.21e-04 | 5254.4ms/step | 11.4 samples/s
|
| 235 |
+
wandb: WARNING Tried to log to step 13730 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 236 |
+
Step 13740/50462 | Loss: 2.6212 | LR: 4.20e-04 | 5247.3ms/step | 11.4 samples/s
|
| 237 |
+
wandb: WARNING Tried to log to step 13740 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 238 |
+
Step 13750/50462 | Loss: 2.6234 | LR: 4.20e-04 | 5247.5ms/step | 11.4 samples/s
|
| 239 |
+
wandb: WARNING Tried to log to step 13750 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 240 |
+
Validation at step 13750: Loss = 2.6969, Perplexity = 14.83
|
| 241 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13750
|
| 242 |
+
Step 13760/50462 | Loss: 2.6068 | LR: 4.20e-04 | 10144.4ms/step | 5.9 samples/s
|
| 243 |
+
wandb: WARNING Tried to log to step 13760 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 244 |
+
Step 13770/50462 | Loss: 2.6058 | LR: 4.20e-04 | 5251.4ms/step | 11.4 samples/s
|
| 245 |
+
wandb: WARNING Tried to log to step 13770 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 246 |
+
Step 13780/50462 | Loss: 2.6062 | LR: 4.20e-04 | 5255.5ms/step | 11.4 samples/s
|
| 247 |
+
wandb: WARNING Tried to log to step 13780 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 248 |
+
Step 13790/50462 | Loss: 2.6155 | LR: 4.20e-04 | 5252.7ms/step | 11.4 samples/s
|
| 249 |
+
wandb: WARNING Tried to log to step 13790 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 250 |
+
Step 13800/50462 | Loss: 2.5953 | LR: 4.20e-04 | 5256.6ms/step | 11.4 samples/s
|
| 251 |
+
wandb: WARNING Tried to log to step 13800 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 252 |
+
Step 13810/50462 | Loss: 2.5958 | LR: 4.20e-04 | 5255.2ms/step | 11.4 samples/s
|
| 253 |
+
wandb: WARNING Tried to log to step 13810 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 254 |
+
Step 13820/50462 | Loss: 2.6024 | LR: 4.19e-04 | 5259.2ms/step | 11.4 samples/s
|
| 255 |
+
wandb: WARNING Tried to log to step 13820 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 256 |
+
Step 13830/50462 | Loss: 2.5974 | LR: 4.19e-04 | 5254.6ms/step | 11.4 samples/s
|
| 257 |
+
wandb: WARNING Tried to log to step 13830 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 258 |
+
Step 13840/50462 | Loss: 2.6033 | LR: 4.19e-04 | 5248.3ms/step | 11.4 samples/s
|
| 259 |
+
wandb: WARNING Tried to log to step 13840 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 260 |
+
Step 13850/50462 | Loss: 2.5919 | LR: 4.19e-04 | 5252.1ms/step | 11.4 samples/s
|
| 261 |
+
wandb: WARNING Tried to log to step 13850 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 262 |
+
Step 13860/50462 | Loss: 2.5948 | LR: 4.19e-04 | 5257.5ms/step | 11.4 samples/s
|
| 263 |
+
wandb: WARNING Tried to log to step 13860 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 264 |
+
Step 13870/50462 | Loss: 2.6241 | LR: 4.19e-04 | 5257.4ms/step | 11.4 samples/s
|
| 265 |
+
wandb: WARNING Tried to log to step 13870 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 266 |
+
Step 13880/50462 | Loss: 2.6233 | LR: 4.19e-04 | 5259.3ms/step | 11.4 samples/s
|
| 267 |
+
wandb: WARNING Tried to log to step 13880 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 268 |
+
Step 13890/50462 | Loss: 2.6220 | LR: 4.19e-04 | 5259.8ms/step | 11.4 samples/s
|
| 269 |
+
wandb: WARNING Tried to log to step 13890 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 270 |
+
Step 13900/50462 | Loss: 2.6202 | LR: 4.19e-04 | 5260.9ms/step | 11.4 samples/s
|
| 271 |
+
wandb: WARNING Tried to log to step 13900 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 272 |
+
Step 13910/50462 | Loss: 2.6004 | LR: 4.18e-04 | 5258.9ms/step | 11.4 samples/s
|
| 273 |
+
wandb: WARNING Tried to log to step 13910 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 274 |
+
Step 13920/50462 | Loss: 2.6166 | LR: 4.18e-04 | 5265.4ms/step | 11.4 samples/s
|
| 275 |
+
wandb: WARNING Tried to log to step 13920 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 276 |
+
Step 13930/50462 | Loss: 2.5898 | LR: 4.18e-04 | 5260.2ms/step | 11.4 samples/s
|
| 277 |
+
wandb: WARNING Tried to log to step 13930 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 278 |
+
Step 13940/50462 | Loss: 2.6027 | LR: 4.18e-04 | 5260.6ms/step | 11.4 samples/s
|
| 279 |
+
wandb: WARNING Tried to log to step 13940 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 280 |
+
Step 13950/50462 | Loss: 2.5709 | LR: 4.18e-04 | 5252.1ms/step | 11.4 samples/s
|
| 281 |
+
wandb: WARNING Tried to log to step 13950 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 282 |
+
Step 13960/50462 | Loss: 2.5982 | LR: 4.18e-04 | 5255.4ms/step | 11.4 samples/s
|
| 283 |
+
wandb: WARNING Tried to log to step 13960 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 284 |
+
Step 13970/50462 | Loss: 2.6064 | LR: 4.18e-04 | 5258.2ms/step | 11.4 samples/s
|
| 285 |
+
wandb: WARNING Tried to log to step 13970 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 286 |
+
Step 13980/50462 | Loss: 2.5810 | LR: 4.18e-04 | 5258.7ms/step | 11.4 samples/s
|
| 287 |
+
wandb: WARNING Tried to log to step 13980 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 288 |
+
Step 13990/50462 | Loss: 2.6062 | LR: 4.17e-04 | 5259.1ms/step | 11.4 samples/s
|
| 289 |
+
wandb: WARNING Tried to log to step 13990 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 290 |
+
Step 14000/50462 | Loss: 2.5894 | LR: 4.17e-04 | 5258.5ms/step | 11.4 samples/s
|
| 291 |
+
wandb: WARNING Tried to log to step 14000 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 292 |
+
Validation at step 14000: Loss = 2.6970, Perplexity = 14.84
|
| 293 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14000
|
| 294 |
+
Step 14010/50462 | Loss: 2.6314 | LR: 4.17e-04 | 10243.6ms/step | 5.9 samples/s
|
| 295 |
+
wandb: WARNING Tried to log to step 14010 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 296 |
+
Step 14020/50462 | Loss: 2.5881 | LR: 4.17e-04 | 5260.5ms/step | 11.4 samples/s
|
| 297 |
+
wandb: WARNING Tried to log to step 14020 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 298 |
+
Step 14030/50462 | Loss: 2.6019 | LR: 4.17e-04 | 5260.0ms/step | 11.4 samples/s
|
| 299 |
+
wandb: WARNING Tried to log to step 14030 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 300 |
+
Step 14040/50462 | Loss: 2.6027 | LR: 4.17e-04 | 5267.5ms/step | 11.4 samples/s
|
| 301 |
+
wandb: WARNING Tried to log to step 14040 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 302 |
+
Step 14050/50462 | Loss: 2.6193 | LR: 4.17e-04 | 5256.1ms/step | 11.4 samples/s
|
| 303 |
+
wandb: WARNING Tried to log to step 14050 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 304 |
+
Step 14060/50462 | Loss: 2.6171 | LR: 4.17e-04 | 5260.8ms/step | 11.4 samples/s
|
| 305 |
+
wandb: WARNING Tried to log to step 14060 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 306 |
+
Step 14070/50462 | Loss: 2.5913 | LR: 4.17e-04 | 5263.1ms/step | 11.4 samples/s
|
| 307 |
+
wandb: WARNING Tried to log to step 14070 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 308 |
+
Step 14080/50462 | Loss: 2.5981 | LR: 4.16e-04 | 5260.8ms/step | 11.4 samples/s
|
| 309 |
+
wandb: WARNING Tried to log to step 14080 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 310 |
+
Step 14090/50462 | Loss: 2.6275 | LR: 4.16e-04 | 5256.8ms/step | 11.4 samples/s
|
| 311 |
+
wandb: WARNING Tried to log to step 14090 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 312 |
+
Step 14100/50462 | Loss: 2.5862 | LR: 4.16e-04 | 5256.7ms/step | 11.4 samples/s
|
| 313 |
+
wandb: WARNING Tried to log to step 14100 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 314 |
+
Step 14110/50462 | Loss: 2.5918 | LR: 4.16e-04 | 5260.0ms/step | 11.4 samples/s
|
| 315 |
+
wandb: WARNING Tried to log to step 14110 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 316 |
+
Step 14120/50462 | Loss: 2.6106 | LR: 4.16e-04 | 5258.7ms/step | 11.4 samples/s
|
| 317 |
+
wandb: WARNING Tried to log to step 14120 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 318 |
+
Step 14130/50462 | Loss: 2.5951 | LR: 4.16e-04 | 5256.7ms/step | 11.4 samples/s
|
| 319 |
+
wandb: WARNING Tried to log to step 14130 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 320 |
+
Step 14140/50462 | Loss: 2.6023 | LR: 4.16e-04 | 5255.5ms/step | 11.4 samples/s
|
| 321 |
+
wandb: WARNING Tried to log to step 14140 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 322 |
+
Step 14150/50462 | Loss: 2.5973 | LR: 4.16e-04 | 5260.8ms/step | 11.4 samples/s
|
| 323 |
+
wandb: WARNING Tried to log to step 14150 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 324 |
+
Step 14160/50462 | Loss: 2.6174 | LR: 4.15e-04 | 5264.1ms/step | 11.4 samples/s
|
| 325 |
+
wandb: WARNING Tried to log to step 14160 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 326 |
+
Step 14170/50462 | Loss: 2.6277 | LR: 4.15e-04 | 5254.9ms/step | 11.4 samples/s
|
| 327 |
+
wandb: WARNING Tried to log to step 14170 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 328 |
+
Step 14180/50462 | Loss: 2.6080 | LR: 4.15e-04 | 5256.5ms/step | 11.4 samples/s
|
| 329 |
+
wandb: WARNING Tried to log to step 14180 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 330 |
+
Step 14190/50462 | Loss: 2.6026 | LR: 4.15e-04 | 5260.0ms/step | 11.4 samples/s
|
| 331 |
+
wandb: WARNING Tried to log to step 14190 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 332 |
+
Step 14200/50462 | Loss: 2.6050 | LR: 4.15e-04 | 5258.2ms/step | 11.4 samples/s
|
| 333 |
+
wandb: WARNING Tried to log to step 14200 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 334 |
+
Step 14210/50462 | Loss: 2.5927 | LR: 4.15e-04 | 5255.1ms/step | 11.4 samples/s
|
| 335 |
+
wandb: WARNING Tried to log to step 14210 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 336 |
+
Step 14220/50462 | Loss: 2.6176 | LR: 4.15e-04 | 5258.9ms/step | 11.4 samples/s
|
| 337 |
+
wandb: WARNING Tried to log to step 14220 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 338 |
+
Step 14230/50462 | Loss: 2.6042 | LR: 4.15e-04 | 5254.0ms/step | 11.4 samples/s
|
| 339 |
+
wandb: WARNING Tried to log to step 14230 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 340 |
+
Step 14240/50462 | Loss: 2.6180 | LR: 4.15e-04 | 5260.6ms/step | 11.4 samples/s
|
| 341 |
+
wandb: WARNING Tried to log to step 14240 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 342 |
+
Step 14250/50462 | Loss: 2.5955 | LR: 4.14e-04 | 5260.3ms/step | 11.4 samples/s
|
| 343 |
+
wandb: WARNING Tried to log to step 14250 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 344 |
+
Validation at step 14250: Loss = 2.6976, Perplexity = 14.84
|
| 345 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14250
|
| 346 |
+
Step 14260/50462 | Loss: 2.5888 | LR: 4.14e-04 | 10241.3ms/step | 5.9 samples/s
|
| 347 |
+
wandb: WARNING Tried to log to step 14260 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 348 |
+
Step 14270/50462 | Loss: 2.5967 | LR: 4.14e-04 | 5254.9ms/step | 11.4 samples/s
|
| 349 |
+
wandb: WARNING Tried to log to step 14270 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 350 |
+
Step 14280/50462 | Loss: 2.5995 | LR: 4.14e-04 | 5258.1ms/step | 11.4 samples/s
|
| 351 |
+
wandb: WARNING Tried to log to step 14280 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 352 |
+
Step 14290/50462 | Loss: 2.5783 | LR: 4.14e-04 | 5260.4ms/step | 11.4 samples/s
|
| 353 |
+
wandb: WARNING Tried to log to step 14290 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 354 |
+
Step 14300/50462 | Loss: 2.6126 | LR: 4.14e-04 | 5259.8ms/step | 11.4 samples/s
|
| 355 |
+
wandb: WARNING Tried to log to step 14300 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 356 |
+
Step 14310/50462 | Loss: 2.5903 | LR: 4.14e-04 | 5261.9ms/step | 11.4 samples/s
|
| 357 |
+
wandb: WARNING Tried to log to step 14310 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 358 |
+
Step 14320/50462 | Loss: 2.5893 | LR: 4.14e-04 | 5261.1ms/step | 11.4 samples/s
|
| 359 |
+
wandb: WARNING Tried to log to step 14320 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 360 |
+
Step 14330/50462 | Loss: 2.6244 | LR: 4.13e-04 | 5259.1ms/step | 11.4 samples/s
|
| 361 |
+
wandb: WARNING Tried to log to step 14330 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 362 |
+
Step 14340/50462 | Loss: 2.5662 | LR: 4.13e-04 | 5263.1ms/step | 11.4 samples/s
|
| 363 |
+
wandb: WARNING Tried to log to step 14340 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 364 |
+
Step 14350/50462 | Loss: 2.5948 | LR: 4.13e-04 | 5259.3ms/step | 11.4 samples/s
|
| 365 |
+
wandb: WARNING Tried to log to step 14350 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 366 |
+
Step 14360/50462 | Loss: 2.6006 | LR: 4.13e-04 | 5260.1ms/step | 11.4 samples/s
|
| 367 |
+
wandb: WARNING Tried to log to step 14360 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 368 |
+
Step 14370/50462 | Loss: 2.5762 | LR: 4.13e-04 | 5263.3ms/step | 11.4 samples/s
|
| 369 |
+
wandb: WARNING Tried to log to step 14370 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 370 |
+
Step 14380/50462 | Loss: 2.5736 | LR: 4.13e-04 | 5264.1ms/step | 11.4 samples/s
|
| 371 |
+
wandb: WARNING Tried to log to step 14380 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 372 |
+
Step 14390/50462 | Loss: 2.5528 | LR: 4.13e-04 | 5263.3ms/step | 11.4 samples/s
|
| 373 |
+
wandb: WARNING Tried to log to step 14390 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 374 |
+
Step 14400/50462 | Loss: 2.6021 | LR: 4.13e-04 | 5263.9ms/step | 11.4 samples/s
|
| 375 |
+
wandb: WARNING Tried to log to step 14400 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 376 |
+
Step 14410/50462 | Loss: 2.6074 | LR: 4.12e-04 | 5258.8ms/step | 11.4 samples/s
|
| 377 |
+
wandb: WARNING Tried to log to step 14410 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 378 |
+
Step 14420/50462 | Loss: 2.5956 | LR: 4.12e-04 | 5263.4ms/step | 11.4 samples/s
|
| 379 |
+
wandb: WARNING Tried to log to step 14420 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 380 |
+
Step 14430/50462 | Loss: 2.5700 | LR: 4.12e-04 | 5259.9ms/step | 11.4 samples/s
|
| 381 |
+
wandb: WARNING Tried to log to step 14430 that is less than the current step 14431. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 382 |
+
Step 14440/50462 | Loss: 2.5924 | LR: 4.12e-04 | 5257.0ms/step | 11.4 samples/s
|
| 383 |
+
Step 14450/50462 | Loss: 2.5934 | LR: 4.12e-04 | 5255.8ms/step | 11.4 samples/s
|
| 384 |
+
Step 14460/50462 | Loss: 2.5592 | LR: 4.12e-04 | 5262.2ms/step | 11.4 samples/s
|
| 385 |
+
Step 14470/50462 | Loss: 2.6006 | LR: 4.12e-04 | 5259.4ms/step | 11.4 samples/s
|
| 386 |
+
Step 14480/50462 | Loss: 2.5971 | LR: 4.12e-04 | 5261.0ms/step | 11.4 samples/s
|
| 387 |
+
Step 14490/50462 | Loss: 2.6076 | LR: 4.12e-04 | 5259.1ms/step | 11.4 samples/s
|
| 388 |
+
Step 14500/50462 | Loss: 2.6152 | LR: 4.11e-04 | 5256.3ms/step | 11.4 samples/s
|
| 389 |
+
Validation at step 14500: Loss = 2.6974, Perplexity = 14.84
|
| 390 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14500
|
| 391 |
+
Step 14510/50462 | Loss: 2.5892 | LR: 4.11e-04 | 10193.0ms/step | 5.9 samples/s
|
| 392 |
+
Step 14520/50462 | Loss: 2.6036 | LR: 4.11e-04 | 5257.9ms/step | 11.4 samples/s
|
| 393 |
+
Step 14530/50462 | Loss: 2.6103 | LR: 4.11e-04 | 5258.8ms/step | 11.4 samples/s
|
| 394 |
+
Step 14540/50462 | Loss: 2.6010 | LR: 4.11e-04 | 5258.9ms/step | 11.4 samples/s
|
| 395 |
+
Step 14550/50462 | Loss: 2.5856 | LR: 4.11e-04 | 5259.0ms/step | 11.4 samples/s
|
| 396 |
+
Step 14560/50462 | Loss: 2.5904 | LR: 4.11e-04 | 5259.4ms/step | 11.4 samples/s
|
| 397 |
+
Step 14570/50462 | Loss: 2.6005 | LR: 4.11e-04 | 5254.1ms/step | 11.4 samples/s
|
| 398 |
+
Step 14580/50462 | Loss: 2.5816 | LR: 4.10e-04 | 5254.4ms/step | 11.4 samples/s
|
| 399 |
+
Step 14590/50462 | Loss: 2.5995 | LR: 4.10e-04 | 5255.8ms/step | 11.4 samples/s
|
| 400 |
+
Step 14600/50462 | Loss: 2.6015 | LR: 4.10e-04 | 5257.9ms/step | 11.4 samples/s
|
| 401 |
+
Step 14610/50462 | Loss: 2.5945 | LR: 4.10e-04 | 5255.4ms/step | 11.4 samples/s
|
| 402 |
+
Step 14620/50462 | Loss: 2.5857 | LR: 4.10e-04 | 5254.0ms/step | 11.4 samples/s
|
| 403 |
+
Step 14630/50462 | Loss: 2.5577 | LR: 4.10e-04 | 5259.2ms/step | 11.4 samples/s
|
| 404 |
+
Step 14640/50462 | Loss: 2.5768 | LR: 4.10e-04 | 5250.5ms/step | 11.4 samples/s
|
| 405 |
+
Step 14650/50462 | Loss: 2.5587 | LR: 4.10e-04 | 5254.8ms/step | 11.4 samples/s
|
| 406 |
+
Step 14660/50462 | Loss: 2.5813 | LR: 4.09e-04 | 5252.0ms/step | 11.4 samples/s
|
| 407 |
+
Step 14670/50462 | Loss: 2.6041 | LR: 4.09e-04 | 5258.1ms/step | 11.4 samples/s
|
| 408 |
+
Step 14680/50462 | Loss: 2.6009 | LR: 4.09e-04 | 5255.6ms/step | 11.4 samples/s
|
| 409 |
+
Step 14690/50462 | Loss: 2.5820 | LR: 4.09e-04 | 5255.0ms/step | 11.4 samples/s
|
| 410 |
+
Step 14700/50462 | Loss: 2.5994 | LR: 4.09e-04 | 5254.7ms/step | 11.4 samples/s
|
| 411 |
+
Step 14710/50462 | Loss: 2.5968 | LR: 4.09e-04 | 5249.7ms/step | 11.4 samples/s
|
| 412 |
+
Step 14720/50462 | Loss: 2.6091 | LR: 4.09e-04 | 5251.2ms/step | 11.4 samples/s
|
| 413 |
+
Step 14730/50462 | Loss: 2.5932 | LR: 4.09e-04 | 5238.0ms/step | 11.5 samples/s
|
| 414 |
+
Step 14740/50462 | Loss: 2.5991 | LR: 4.08e-04 | 5231.4ms/step | 11.5 samples/s
|
| 415 |
+
Step 14750/50462 | Loss: 2.5984 | LR: 4.08e-04 | 5236.3ms/step | 11.5 samples/s
|
| 416 |
+
Validation at step 14750: Loss = 2.6969, Perplexity = 14.83
|
| 417 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-14750
|
| 418 |
+
Step 14760/50462 | Loss: 2.5961 | LR: 4.08e-04 | 10177.7ms/step | 5.9 samples/s
|
| 419 |
+
Step 14770/50462 | Loss: 2.6188 | LR: 4.08e-04 | 5254.2ms/step | 11.4 samples/s
|
| 420 |
+
Step 14780/50462 | Loss: 2.6042 | LR: 4.08e-04 | 5251.1ms/step | 11.4 samples/s
|
| 421 |
+
Step 14790/50462 | Loss: 2.5840 | LR: 4.08e-04 | 5258.2ms/step | 11.4 samples/s
|
| 422 |
+
Step 14800/50462 | Loss: 2.5821 | LR: 4.08e-04 | 5258.7ms/step | 11.4 samples/s
|
| 423 |
+
Step 14810/50462 | Loss: 2.6035 | LR: 4.08e-04 | 5253.7ms/step | 11.4 samples/s
|
| 424 |
+
Step 14820/50462 | Loss: 2.5770 | LR: 4.08e-04 | 5262.4ms/step | 11.4 samples/s
|
| 425 |
+
Step 14830/50462 | Loss: 2.5833 | LR: 4.07e-04 | 5259.5ms/step | 11.4 samples/s
|
| 426 |
+
Step 14840/50462 | Loss: 2.6310 | LR: 4.07e-04 | 5259.3ms/step | 11.4 samples/s
|
| 427 |
+
Step 14850/50462 | Loss: 2.6318 | LR: 4.07e-04 | 5260.0ms/step | 11.4 samples/s
|
| 428 |
+
Step 14860/50462 | Loss: 2.6036 | LR: 4.07e-04 | 5258.6ms/step | 11.4 samples/s
|
| 429 |
+
Step 14870/50462 | Loss: 2.6005 | LR: 4.07e-04 | 5261.1ms/step | 11.4 samples/s
|
| 430 |
+
Step 14880/50462 | Loss: 2.5679 | LR: 4.07e-04 | 5259.7ms/step | 11.4 samples/s
|
| 431 |
+
Step 14890/50462 | Loss: 2.5928 | LR: 4.07e-04 | 5258.9ms/step | 11.4 samples/s
|
| 432 |
+
Step 14900/50462 | Loss: 2.6090 | LR: 4.07e-04 | 5256.3ms/step | 11.4 samples/s
|
| 433 |
+
Step 14910/50462 | Loss: 2.5979 | LR: 4.06e-04 | 5254.5ms/step | 11.4 samples/s
|
| 434 |
+
Step 14920/50462 | Loss: 2.5907 | LR: 4.06e-04 | 5252.1ms/step | 11.4 samples/s
|
| 435 |
+
Step 14930/50462 | Loss: 2.5941 | LR: 4.06e-04 | 5257.0ms/step | 11.4 samples/s
|
| 436 |
+
Step 14940/50462 | Loss: 2.5892 | LR: 4.06e-04 | 5255.4ms/step | 11.4 samples/s
|
| 437 |
+
Step 14950/50462 | Loss: 2.5902 | LR: 4.06e-04 | 5252.3ms/step | 11.4 samples/s
|
| 438 |
+
Step 14960/50462 | Loss: 2.6008 | LR: 4.06e-04 | 5255.7ms/step | 11.4 samples/s
|
| 439 |
+
Step 14970/50462 | Loss: 2.5571 | LR: 4.06e-04 | 5253.0ms/step | 11.4 samples/s
|
| 440 |
+
Step 14980/50462 | Loss: 2.5836 | LR: 4.06e-04 | 5252.5ms/step | 11.4 samples/s
|
| 441 |
+
Step 14990/50462 | Loss: 2.5911 | LR: 4.05e-04 | 5254.0ms/step | 11.4 samples/s
|
| 442 |
+
Step 15000/50462 | Loss: 2.5963 | LR: 4.05e-04 | 5254.1ms/step | 11.4 samples/s
|
| 443 |
+
Validation at step 15000: Loss = 2.6955, Perplexity = 14.81
|
| 444 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15000
|
| 445 |
+
Step 15010/50462 | Loss: 2.5961 | LR: 4.05e-04 | 10196.7ms/step | 5.9 samples/s
|
| 446 |
+
Step 15020/50462 | Loss: 2.5905 | LR: 4.05e-04 | 5249.4ms/step | 11.4 samples/s
|
| 447 |
+
Step 15030/50462 | Loss: 2.5922 | LR: 4.05e-04 | 5256.5ms/step | 11.4 samples/s
|
| 448 |
+
Step 15040/50462 | Loss: 2.5921 | LR: 4.05e-04 | 5252.1ms/step | 11.4 samples/s
|
| 449 |
+
Step 15050/50462 | Loss: 2.6171 | LR: 4.05e-04 | 5251.7ms/step | 11.4 samples/s
|
| 450 |
+
Step 15060/50462 | Loss: 2.5734 | LR: 4.05e-04 | 5255.5ms/step | 11.4 samples/s
|
| 451 |
+
Step 15070/50462 | Loss: 2.5866 | LR: 4.04e-04 | 5254.1ms/step | 11.4 samples/s
|
| 452 |
+
Step 15080/50462 | Loss: 2.6219 | LR: 4.04e-04 | 5259.6ms/step | 11.4 samples/s
|
| 453 |
+
Step 15090/50462 | Loss: 2.5759 | LR: 4.04e-04 | 5252.8ms/step | 11.4 samples/s
|
| 454 |
+
Step 15100/50462 | Loss: 2.6238 | LR: 4.04e-04 | 5253.0ms/step | 11.4 samples/s
|
| 455 |
+
Step 15110/50462 | Loss: 2.5872 | LR: 4.04e-04 | 5249.0ms/step | 11.4 samples/s
|
| 456 |
+
Step 15120/50462 | Loss: 2.6010 | LR: 4.04e-04 | 5250.9ms/step | 11.4 samples/s
|
| 457 |
+
Step 15130/50462 | Loss: 2.5726 | LR: 4.04e-04 | 5252.6ms/step | 11.4 samples/s
|
| 458 |
+
Step 15140/50462 | Loss: 2.6049 | LR: 4.04e-04 | 5256.4ms/step | 11.4 samples/s
|
| 459 |
+
Step 15150/50462 | Loss: 2.5988 | LR: 4.03e-04 | 5256.9ms/step | 11.4 samples/s
|
| 460 |
+
Step 15160/50462 | Loss: 2.5711 | LR: 4.03e-04 | 5249.4ms/step | 11.4 samples/s
|
| 461 |
+
Step 15170/50462 | Loss: 2.6170 | LR: 4.03e-04 | 5251.6ms/step | 11.4 samples/s
|
| 462 |
+
Step 15180/50462 | Loss: 2.6008 | LR: 4.03e-04 | 5253.6ms/step | 11.4 samples/s
|
| 463 |
+
Step 15190/50462 | Loss: 2.5808 | LR: 4.03e-04 | 5249.2ms/step | 11.4 samples/s
|
| 464 |
+
Step 15200/50462 | Loss: 2.5941 | LR: 4.03e-04 | 5256.5ms/step | 11.4 samples/s
|
| 465 |
+
Step 15210/50462 | Loss: 2.6047 | LR: 4.03e-04 | 5251.9ms/step | 11.4 samples/s
|
| 466 |
+
Step 15220/50462 | Loss: 2.5929 | LR: 4.03e-04 | 5249.6ms/step | 11.4 samples/s
|
| 467 |
+
Step 15230/50462 | Loss: 2.5720 | LR: 4.02e-04 | 5250.9ms/step | 11.4 samples/s
|
| 468 |
+
Step 15240/50462 | Loss: 2.6031 | LR: 4.02e-04 | 5253.3ms/step | 11.4 samples/s
|
| 469 |
+
Step 15250/50462 | Loss: 2.5804 | LR: 4.02e-04 | 5256.4ms/step | 11.4 samples/s
|
| 470 |
+
Validation at step 15250: Loss = 2.6934, Perplexity = 14.78
|
| 471 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15250
|
| 472 |
+
Step 15260/50462 | Loss: 2.6015 | LR: 4.02e-04 | 10149.0ms/step | 5.9 samples/s
|
| 473 |
+
Step 15270/50462 | Loss: 2.6012 | LR: 4.02e-04 | 5251.1ms/step | 11.4 samples/s
|
| 474 |
+
Step 15280/50462 | Loss: 2.5760 | LR: 4.02e-04 | 5255.9ms/step | 11.4 samples/s
|
| 475 |
+
Step 15290/50462 | Loss: 2.5770 | LR: 4.02e-04 | 5255.8ms/step | 11.4 samples/s
|
| 476 |
+
Step 15300/50462 | Loss: 2.5675 | LR: 4.02e-04 | 5257.3ms/step | 11.4 samples/s
|
| 477 |
+
Step 15310/50462 | Loss: 2.6076 | LR: 4.01e-04 | 5254.8ms/step | 11.4 samples/s
|
| 478 |
+
Step 15320/50462 | Loss: 2.5929 | LR: 4.01e-04 | 5255.1ms/step | 11.4 samples/s
|
| 479 |
+
Step 15330/50462 | Loss: 2.5974 | LR: 4.01e-04 | 5254.5ms/step | 11.4 samples/s
|
| 480 |
+
Step 15340/50462 | Loss: 2.5763 | LR: 4.01e-04 | 5260.8ms/step | 11.4 samples/s
|
| 481 |
+
Step 15350/50462 | Loss: 2.5890 | LR: 4.01e-04 | 5262.7ms/step | 11.4 samples/s
|
| 482 |
+
Step 15360/50462 | Loss: 2.5774 | LR: 4.01e-04 | 5263.4ms/step | 11.4 samples/s
|
| 483 |
+
Step 15370/50462 | Loss: 2.6042 | LR: 4.01e-04 | 5256.6ms/step | 11.4 samples/s
|
| 484 |
+
Step 15380/50462 | Loss: 2.5976 | LR: 4.01e-04 | 5251.8ms/step | 11.4 samples/s
|
| 485 |
+
Step 15390/50462 | Loss: 2.5890 | LR: 4.00e-04 | 5250.7ms/step | 11.4 samples/s
|
| 486 |
+
Step 15400/50462 | Loss: 2.6028 | LR: 4.00e-04 | 5251.2ms/step | 11.4 samples/s
|
| 487 |
+
Step 15410/50462 | Loss: 2.5913 | LR: 4.00e-04 | 5250.8ms/step | 11.4 samples/s
|
| 488 |
+
Step 15420/50462 | Loss: 2.5939 | LR: 4.00e-04 | 5253.8ms/step | 11.4 samples/s
|
| 489 |
+
Step 15430/50462 | Loss: 2.5854 | LR: 4.00e-04 | 5251.9ms/step | 11.4 samples/s
|
| 490 |
+
Step 15440/50462 | Loss: 2.5980 | LR: 4.00e-04 | 5254.1ms/step | 11.4 samples/s
|
| 491 |
+
Step 15450/50462 | Loss: 2.5673 | LR: 4.00e-04 | 5254.7ms/step | 11.4 samples/s
|
| 492 |
+
Step 15460/50462 | Loss: 2.5870 | LR: 4.00e-04 | 5255.9ms/step | 11.4 samples/s
|
| 493 |
+
Step 15470/50462 | Loss: 2.5876 | LR: 3.99e-04 | 5256.6ms/step | 11.4 samples/s
|
| 494 |
+
Step 15480/50462 | Loss: 2.6031 | LR: 3.99e-04 | 5254.3ms/step | 11.4 samples/s
|
| 495 |
+
Step 15490/50462 | Loss: 2.5735 | LR: 3.99e-04 | 5252.9ms/step | 11.4 samples/s
|
| 496 |
+
Step 15500/50462 | Loss: 2.5920 | LR: 3.99e-04 | 5251.3ms/step | 11.4 samples/s
|
| 497 |
+
Validation at step 15500: Loss = 2.6935, Perplexity = 14.78
|
| 498 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15500
|
| 499 |
+
Step 15510/50462 | Loss: 2.5917 | LR: 3.99e-04 | 10104.6ms/step | 5.9 samples/s
|
| 500 |
+
Step 15520/50462 | Loss: 2.5826 | LR: 3.99e-04 | 5249.9ms/step | 11.4 samples/s
|
| 501 |
+
Step 15530/50462 | Loss: 2.6022 | LR: 3.99e-04 | 5254.4ms/step | 11.4 samples/s
|
| 502 |
+
Step 15540/50462 | Loss: 2.5862 | LR: 3.99e-04 | 5254.4ms/step | 11.4 samples/s
|
| 503 |
+
Step 15550/50462 | Loss: 2.5800 | LR: 3.98e-04 | 5256.5ms/step | 11.4 samples/s
|
| 504 |
+
Step 15560/50462 | Loss: 2.5930 | LR: 3.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 505 |
+
Step 15570/50462 | Loss: 2.6109 | LR: 3.98e-04 | 5262.4ms/step | 11.4 samples/s
|
| 506 |
+
Step 15580/50462 | Loss: 2.5938 | LR: 3.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 507 |
+
Step 15590/50462 | Loss: 2.5845 | LR: 3.98e-04 | 5258.1ms/step | 11.4 samples/s
|
| 508 |
+
Step 15600/50462 | Loss: 2.5621 | LR: 3.98e-04 | 5260.0ms/step | 11.4 samples/s
|
| 509 |
+
Step 15610/50462 | Loss: 2.5745 | LR: 3.98e-04 | 5254.4ms/step | 11.4 samples/s
|
| 510 |
+
Step 15620/50462 | Loss: 2.5914 | LR: 3.98e-04 | 5256.8ms/step | 11.4 samples/s
|
| 511 |
+
Step 15630/50462 | Loss: 2.6048 | LR: 3.97e-04 | 5254.9ms/step | 11.4 samples/s
|
| 512 |
+
Step 15640/50462 | Loss: 2.5648 | LR: 3.97e-04 | 5260.7ms/step | 11.4 samples/s
|
| 513 |
+
Step 15650/50462 | Loss: 2.5650 | LR: 3.97e-04 | 5258.1ms/step | 11.4 samples/s
|
| 514 |
+
Step 15660/50462 | Loss: 2.5839 | LR: 3.97e-04 | 5258.1ms/step | 11.4 samples/s
|
| 515 |
+
Step 15670/50462 | Loss: 2.6103 | LR: 3.97e-04 | 5254.7ms/step | 11.4 samples/s
|
| 516 |
+
Step 15680/50462 | Loss: 2.5890 | LR: 3.97e-04 | 5253.8ms/step | 11.4 samples/s
|
| 517 |
+
Step 15690/50462 | Loss: 2.6038 | LR: 3.97e-04 | 5248.8ms/step | 11.4 samples/s
|
| 518 |
+
Step 15700/50462 | Loss: 2.5974 | LR: 3.96e-04 | 5251.9ms/step | 11.4 samples/s
|
| 519 |
+
Step 15710/50462 | Loss: 2.5917 | LR: 3.96e-04 | 5254.6ms/step | 11.4 samples/s
|
| 520 |
+
Step 15720/50462 | Loss: 2.6000 | LR: 3.96e-04 | 5260.3ms/step | 11.4 samples/s
|
| 521 |
+
Step 15730/50462 | Loss: 2.5923 | LR: 3.96e-04 | 5257.7ms/step | 11.4 samples/s
|
| 522 |
+
Step 15740/50462 | Loss: 2.5743 | LR: 3.96e-04 | 5260.8ms/step | 11.4 samples/s
|
| 523 |
+
Step 15750/50462 | Loss: 2.6047 | LR: 3.96e-04 | 5261.8ms/step | 11.4 samples/s
|
| 524 |
+
Validation at step 15750: Loss = 2.6929, Perplexity = 14.78
|
| 525 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-15750
|
| 526 |
+
Step 15760/50462 | Loss: 2.5691 | LR: 3.96e-04 | 10106.5ms/step | 5.9 samples/s
|
| 527 |
+
Step 15770/50462 | Loss: 2.5990 | LR: 3.96e-04 | 5237.2ms/step | 11.5 samples/s
|
| 528 |
+
Step 15780/50462 | Loss: 2.5820 | LR: 3.95e-04 | 5238.2ms/step | 11.5 samples/s
|
| 529 |
+
Step 15790/50462 | Loss: 2.6089 | LR: 3.95e-04 | 5238.5ms/step | 11.5 samples/s
|
| 530 |
+
Step 15800/50462 | Loss: 2.6076 | LR: 3.95e-04 | 5233.4ms/step | 11.5 samples/s
|
| 531 |
+
Step 15810/50462 | Loss: 2.5743 | LR: 3.95e-04 | 5231.5ms/step | 11.5 samples/s
|
| 532 |
+
Step 15820/50462 | Loss: 2.5922 | LR: 3.95e-04 | 5232.6ms/step | 11.5 samples/s
|
| 533 |
+
Step 15830/50462 | Loss: 2.5755 | LR: 3.95e-04 | 5235.4ms/step | 11.5 samples/s
|
| 534 |
+
Step 15840/50462 | Loss: 2.5793 | LR: 3.95e-04 | 5234.9ms/step | 11.5 samples/s
|
| 535 |
+
Step 15850/50462 | Loss: 2.5882 | LR: 3.95e-04 | 5234.8ms/step | 11.5 samples/s
|
| 536 |
+
Step 15860/50462 | Loss: 2.5818 | LR: 3.94e-04 | 5238.2ms/step | 11.5 samples/s
|
| 537 |
+
Step 15870/50462 | Loss: 2.5719 | LR: 3.94e-04 | 5238.2ms/step | 11.5 samples/s
|
| 538 |
+
Step 15880/50462 | Loss: 2.5906 | LR: 3.94e-04 | 5241.1ms/step | 11.4 samples/s
|
| 539 |
+
Step 15890/50462 | Loss: 2.5650 | LR: 3.94e-04 | 5242.1ms/step | 11.4 samples/s
|
| 540 |
+
Step 15900/50462 | Loss: 2.5709 | LR: 3.94e-04 | 5242.5ms/step | 11.4 samples/s
|
| 541 |
+
Step 15910/50462 | Loss: 2.5951 | LR: 3.94e-04 | 5237.8ms/step | 11.5 samples/s
|
| 542 |
+
Step 15920/50462 | Loss: 2.5806 | LR: 3.94e-04 | 5235.6ms/step | 11.5 samples/s
|
| 543 |
+
Step 15930/50462 | Loss: 2.5731 | LR: 3.94e-04 | 5232.9ms/step | 11.5 samples/s
|
| 544 |
+
Step 15940/50462 | Loss: 2.5848 | LR: 3.93e-04 | 5239.7ms/step | 11.5 samples/s
|
| 545 |
+
Step 15950/50462 | Loss: 2.6084 | LR: 3.93e-04 | 5235.3ms/step | 11.5 samples/s
|
| 546 |
+
Step 15960/50462 | Loss: 2.5868 | LR: 3.93e-04 | 5235.1ms/step | 11.5 samples/s
|
| 547 |
+
Step 15970/50462 | Loss: 2.5861 | LR: 3.93e-04 | 5229.6ms/step | 11.5 samples/s
|
| 548 |
+
Step 15980/50462 | Loss: 2.5803 | LR: 3.93e-04 | 5242.2ms/step | 11.4 samples/s
|
| 549 |
+
Step 15990/50462 | Loss: 2.6005 | LR: 3.93e-04 | 5231.1ms/step | 11.5 samples/s
|
| 550 |
+
Step 16000/50462 | Loss: 2.5985 | LR: 3.93e-04 | 5230.0ms/step | 11.5 samples/s
|
| 551 |
+
Validation at step 16000: Loss = 2.6900, Perplexity = 14.73
|
| 552 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16000
|
| 553 |
+
Step 16010/50462 | Loss: 2.5732 | LR: 3.92e-04 | 10152.8ms/step | 5.9 samples/s
|
| 554 |
+
Step 16020/50462 | Loss: 2.5728 | LR: 3.92e-04 | 5257.9ms/step | 11.4 samples/s
|
| 555 |
+
Step 16030/50462 | Loss: 2.6043 | LR: 3.92e-04 | 5256.7ms/step | 11.4 samples/s
|
| 556 |
+
Step 16040/50462 | Loss: 2.5809 | LR: 3.92e-04 | 5244.4ms/step | 11.4 samples/s
|
| 557 |
+
Step 16050/50462 | Loss: 2.5976 | LR: 3.92e-04 | 5238.3ms/step | 11.5 samples/s
|
| 558 |
+
Step 16060/50462 | Loss: 2.5926 | LR: 3.92e-04 | 5242.6ms/step | 11.4 samples/s
|
| 559 |
+
Step 16070/50462 | Loss: 2.6094 | LR: 3.92e-04 | 5240.2ms/step | 11.4 samples/s
|
| 560 |
+
Step 16080/50462 | Loss: 2.5933 | LR: 3.92e-04 | 5238.8ms/step | 11.5 samples/s
|
| 561 |
+
Step 16090/50462 | Loss: 2.5859 | LR: 3.91e-04 | 5245.6ms/step | 11.4 samples/s
|
| 562 |
+
Step 16100/50462 | Loss: 2.5837 | LR: 3.91e-04 | 5238.2ms/step | 11.5 samples/s
|
| 563 |
+
Step 16110/50462 | Loss: 2.5723 | LR: 3.91e-04 | 5235.5ms/step | 11.5 samples/s
|
| 564 |
+
Step 16120/50462 | Loss: 2.5728 | LR: 3.91e-04 | 5228.8ms/step | 11.5 samples/s
|
| 565 |
+
Step 16130/50462 | Loss: 2.5771 | LR: 3.91e-04 | 5226.0ms/step | 11.5 samples/s
|
| 566 |
+
Step 16140/50462 | Loss: 2.5951 | LR: 3.91e-04 | 5232.3ms/step | 11.5 samples/s
|
| 567 |
+
Step 16150/50462 | Loss: 2.5761 | LR: 3.91e-04 | 5231.6ms/step | 11.5 samples/s
|
| 568 |
+
Step 16160/50462 | Loss: 2.5733 | LR: 3.91e-04 | 5238.3ms/step | 11.5 samples/s
|
| 569 |
+
Step 16170/50462 | Loss: 2.5755 | LR: 3.90e-04 | 5230.3ms/step | 11.5 samples/s
|
| 570 |
+
Step 16180/50462 | Loss: 2.5875 | LR: 3.90e-04 | 5245.1ms/step | 11.4 samples/s
|
| 571 |
+
Step 16190/50462 | Loss: 2.5954 | LR: 3.90e-04 | 5230.6ms/step | 11.5 samples/s
|
| 572 |
+
Step 16200/50462 | Loss: 2.5650 | LR: 3.90e-04 | 5229.7ms/step | 11.5 samples/s
|
| 573 |
+
Step 16210/50462 | Loss: 2.5966 | LR: 3.90e-04 | 5225.9ms/step | 11.5 samples/s
|
| 574 |
+
Step 16220/50462 | Loss: 2.5780 | LR: 3.90e-04 | 5227.6ms/step | 11.5 samples/s
|
| 575 |
+
Step 16230/50462 | Loss: 2.5901 | LR: 3.90e-04 | 5229.8ms/step | 11.5 samples/s
|
| 576 |
+
Step 16240/50462 | Loss: 2.5868 | LR: 3.89e-04 | 5230.4ms/step | 11.5 samples/s
|
| 577 |
+
Step 16250/50462 | Loss: 2.5653 | LR: 3.89e-04 | 5229.8ms/step | 11.5 samples/s
|
| 578 |
+
Validation at step 16250: Loss = 2.6891, Perplexity = 14.72
|
| 579 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16250
|
| 580 |
+
Step 16260/50462 | Loss: 2.5994 | LR: 3.89e-04 | 10099.7ms/step | 5.9 samples/s
|
| 581 |
+
Step 16270/50462 | Loss: 2.5961 | LR: 3.89e-04 | 5231.1ms/step | 11.5 samples/s
|
| 582 |
+
Step 16280/50462 | Loss: 2.5651 | LR: 3.89e-04 | 5234.7ms/step | 11.5 samples/s
|
| 583 |
+
Step 16290/50462 | Loss: 2.5972 | LR: 3.89e-04 | 5243.3ms/step | 11.4 samples/s
|
| 584 |
+
Step 16300/50462 | Loss: 2.6193 | LR: 3.89e-04 | 5233.5ms/step | 11.5 samples/s
|
| 585 |
+
Step 16310/50462 | Loss: 2.6008 | LR: 3.89e-04 | 5237.5ms/step | 11.5 samples/s
|
| 586 |
+
Step 16320/50462 | Loss: 2.5744 | LR: 3.88e-04 | 5229.5ms/step | 11.5 samples/s
|
| 587 |
+
Step 16330/50462 | Loss: 2.6035 | LR: 3.88e-04 | 5239.6ms/step | 11.5 samples/s
|
| 588 |
+
Step 16340/50462 | Loss: 2.6097 | LR: 3.88e-04 | 5231.0ms/step | 11.5 samples/s
|
| 589 |
+
Step 16350/50462 | Loss: 2.5874 | LR: 3.88e-04 | 5229.2ms/step | 11.5 samples/s
|
| 590 |
+
Step 16360/50462 | Loss: 2.5836 | LR: 3.88e-04 | 5225.2ms/step | 11.5 samples/s
|
| 591 |
+
Step 16370/50462 | Loss: 2.6065 | LR: 3.88e-04 | 5226.6ms/step | 11.5 samples/s
|
| 592 |
+
Step 16380/50462 | Loss: 2.5605 | LR: 3.88e-04 | 5235.1ms/step | 11.5 samples/s
|
| 593 |
+
Step 16390/50462 | Loss: 2.5982 | LR: 3.88e-04 | 5224.3ms/step | 11.5 samples/s
|
| 594 |
+
Step 16400/50462 | Loss: 2.5626 | LR: 3.87e-04 | 5230.9ms/step | 11.5 samples/s
|
| 595 |
+
Step 16410/50462 | Loss: 2.5759 | LR: 3.87e-04 | 5231.7ms/step | 11.5 samples/s
|
| 596 |
+
Step 16420/50462 | Loss: 2.5934 | LR: 3.87e-04 | 5235.5ms/step | 11.5 samples/s
|
| 597 |
+
Step 16430/50462 | Loss: 2.5680 | LR: 3.87e-04 | 5232.4ms/step | 11.5 samples/s
|
| 598 |
+
Step 16440/50462 | Loss: 2.5954 | LR: 3.87e-04 | 5227.1ms/step | 11.5 samples/s
|
| 599 |
+
Step 16450/50462 | Loss: 2.6783 | LR: 3.87e-04 | 5226.5ms/step | 11.5 samples/s
|
| 600 |
+
Step 16460/50462 | Loss: 2.6648 | LR: 3.87e-04 | 5228.0ms/step | 11.5 samples/s
|
| 601 |
+
Step 16470/50462 | Loss: 2.6929 | LR: 3.86e-04 | 5228.7ms/step | 11.5 samples/s
|
| 602 |
+
Step 16480/50462 | Loss: 2.6569 | LR: 3.86e-04 | 5235.1ms/step | 11.5 samples/s
|
| 603 |
+
Step 16490/50462 | Loss: 2.6710 | LR: 3.86e-04 | 5233.3ms/step | 11.5 samples/s
|
| 604 |
+
Step 16500/50462 | Loss: 2.6969 | LR: 3.86e-04 | 5233.0ms/step | 11.5 samples/s
|
| 605 |
+
Validation at step 16500: Loss = 2.6783, Perplexity = 14.56
|
| 606 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16500
|
| 607 |
+
Step 16510/50462 | Loss: 2.6662 | LR: 3.86e-04 | 10141.6ms/step | 5.9 samples/s
|
| 608 |
+
Step 16520/50462 | Loss: 2.6532 | LR: 3.86e-04 | 5232.3ms/step | 11.5 samples/s
|
| 609 |
+
Step 16530/50462 | Loss: 2.6691 | LR: 3.86e-04 | 5230.9ms/step | 11.5 samples/s
|
| 610 |
+
Step 16540/50462 | Loss: 2.6758 | LR: 3.86e-04 | 5234.3ms/step | 11.5 samples/s
|
| 611 |
+
Step 16550/50462 | Loss: 2.6622 | LR: 3.85e-04 | 5228.7ms/step | 11.5 samples/s
|
| 612 |
+
Step 16560/50462 | Loss: 2.6443 | LR: 3.85e-04 | 5228.2ms/step | 11.5 samples/s
|
| 613 |
+
Step 16570/50462 | Loss: 2.6606 | LR: 3.85e-04 | 5228.2ms/step | 11.5 samples/s
|
| 614 |
+
Step 16580/50462 | Loss: 2.6622 | LR: 3.85e-04 | 5229.1ms/step | 11.5 samples/s
|
| 615 |
+
Step 16590/50462 | Loss: 2.6694 | LR: 3.85e-04 | 5229.4ms/step | 11.5 samples/s
|
| 616 |
+
Step 16600/50462 | Loss: 2.6484 | LR: 3.85e-04 | 5220.8ms/step | 11.5 samples/s
|
| 617 |
+
Step 16610/50462 | Loss: 2.6436 | LR: 3.85e-04 | 5234.1ms/step | 11.5 samples/s
|
| 618 |
+
Step 16620/50462 | Loss: 2.6728 | LR: 3.84e-04 | 5228.4ms/step | 11.5 samples/s
|
| 619 |
+
Step 16630/50462 | Loss: 2.6473 | LR: 3.84e-04 | 5219.3ms/step | 11.5 samples/s
|
| 620 |
+
Step 16640/50462 | Loss: 2.6741 | LR: 3.84e-04 | 5228.0ms/step | 11.5 samples/s
|
| 621 |
+
Step 16650/50462 | Loss: 2.6729 | LR: 3.84e-04 | 5230.1ms/step | 11.5 samples/s
|
| 622 |
+
Step 16660/50462 | Loss: 2.6525 | LR: 3.84e-04 | 5229.4ms/step | 11.5 samples/s
|
| 623 |
+
Step 16670/50462 | Loss: 2.6642 | LR: 3.84e-04 | 5232.4ms/step | 11.5 samples/s
|
| 624 |
+
Step 16680/50462 | Loss: 2.6518 | LR: 3.84e-04 | 5233.6ms/step | 11.5 samples/s
|
| 625 |
+
Step 16690/50462 | Loss: 2.6672 | LR: 3.84e-04 | 5233.7ms/step | 11.5 samples/s
|
| 626 |
+
Step 16700/50462 | Loss: 2.6535 | LR: 3.83e-04 | 5230.1ms/step | 11.5 samples/s
|
| 627 |
+
Step 16710/50462 | Loss: 2.6896 | LR: 3.83e-04 | 5232.8ms/step | 11.5 samples/s
|
| 628 |
+
Step 16720/50462 | Loss: 2.6610 | LR: 3.83e-04 | 5232.1ms/step | 11.5 samples/s
|
| 629 |
+
Step 16730/50462 | Loss: 2.6708 | LR: 3.83e-04 | 5233.8ms/step | 11.5 samples/s
|
| 630 |
+
Step 16740/50462 | Loss: 2.6713 | LR: 3.83e-04 | 5233.3ms/step | 11.5 samples/s
|
| 631 |
+
Step 16750/50462 | Loss: 2.6428 | LR: 3.83e-04 | 5230.7ms/step | 11.5 samples/s
|
| 632 |
+
Validation at step 16750: Loss = 2.6730, Perplexity = 14.48
|
| 633 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-16750
|
| 634 |
+
Step 16760/50462 | Loss: 2.6969 | LR: 3.83e-04 | 10165.9ms/step | 5.9 samples/s
|
| 635 |
+
Step 16770/50462 | Loss: 2.6650 | LR: 3.82e-04 | 5256.1ms/step | 11.4 samples/s
|
| 636 |
+
Step 16780/50462 | Loss: 2.6560 | LR: 3.82e-04 | 5257.2ms/step | 11.4 samples/s
|
| 637 |
+
Step 16790/50462 | Loss: 2.6474 | LR: 3.82e-04 | 5257.1ms/step | 11.4 samples/s
|
| 638 |
+
Step 16800/50462 | Loss: 2.6592 | LR: 3.82e-04 | 5255.6ms/step | 11.4 samples/s
|
| 639 |
+
Step 16810/50462 | Loss: 2.6480 | LR: 3.82e-04 | 5256.7ms/step | 11.4 samples/s
|
| 640 |
+
Step 16820/50462 | Loss: 2.6679 | LR: 3.82e-04 | 5257.8ms/step | 11.4 samples/s
|
| 641 |
+
Step 16830/50462 | Loss: 2.6607 | LR: 3.82e-04 | 5256.9ms/step | 11.4 samples/s
|
| 642 |
+
Step 16840/50462 | Loss: 2.6475 | LR: 3.82e-04 | 5259.6ms/step | 11.4 samples/s
|
| 643 |
+
Step 16850/50462 | Loss: 2.6324 | LR: 3.81e-04 | 5258.2ms/step | 11.4 samples/s
|
| 644 |
+
Step 16860/50462 | Loss: 2.6650 | LR: 3.81e-04 | 5255.6ms/step | 11.4 samples/s
|
| 645 |
+
Step 16870/50462 | Loss: 2.6795 | LR: 3.81e-04 | 5251.4ms/step | 11.4 samples/s
|
| 646 |
+
Step 16880/50462 | Loss: 2.6527 | LR: 3.81e-04 | 5248.5ms/step | 11.4 samples/s
|
| 647 |
+
Step 16890/50462 | Loss: 2.6748 | LR: 3.81e-04 | 5254.9ms/step | 11.4 samples/s
|
| 648 |
+
Step 16900/50462 | Loss: 2.6418 | LR: 3.81e-04 | 5254.9ms/step | 11.4 samples/s
|
| 649 |
+
Step 16910/50462 | Loss: 2.6807 | LR: 3.81e-04 | 5254.5ms/step | 11.4 samples/s
|
| 650 |
+
Step 16920/50462 | Loss: 2.6627 | LR: 3.80e-04 | 5245.8ms/step | 11.4 samples/s
|
| 651 |
+
Step 16930/50462 | Loss: 2.6625 | LR: 3.80e-04 | 5251.4ms/step | 11.4 samples/s
|
| 652 |
+
Step 16940/50462 | Loss: 2.6637 | LR: 3.80e-04 | 5251.6ms/step | 11.4 samples/s
|
| 653 |
+
Step 16950/50462 | Loss: 2.6541 | LR: 3.80e-04 | 5251.6ms/step | 11.4 samples/s
|
| 654 |
+
Step 16960/50462 | Loss: 2.6514 | LR: 3.80e-04 | 5250.5ms/step | 11.4 samples/s
|
| 655 |
+
Step 16970/50462 | Loss: 2.6738 | LR: 3.80e-04 | 5250.0ms/step | 11.4 samples/s
|
| 656 |
+
Step 16980/50462 | Loss: 2.6814 | LR: 3.80e-04 | 5250.4ms/step | 11.4 samples/s
|
| 657 |
+
Step 16990/50462 | Loss: 2.6625 | LR: 3.79e-04 | 5251.5ms/step | 11.4 samples/s
|
| 658 |
+
Step 17000/50462 | Loss: 2.6530 | LR: 3.79e-04 | 5250.1ms/step | 11.4 samples/s
|
| 659 |
+
Validation at step 17000: Loss = 2.6670, Perplexity = 14.40
|
| 660 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17000
|
| 661 |
+
Step 17010/50462 | Loss: 2.6488 | LR: 3.79e-04 | 10172.4ms/step | 5.9 samples/s
|
| 662 |
+
Step 17020/50462 | Loss: 2.6560 | LR: 3.79e-04 | 5252.0ms/step | 11.4 samples/s
|
| 663 |
+
Step 17030/50462 | Loss: 2.6591 | LR: 3.79e-04 | 5253.5ms/step | 11.4 samples/s
|
| 664 |
+
Step 17040/50462 | Loss: 2.6780 | LR: 3.79e-04 | 5250.9ms/step | 11.4 samples/s
|
| 665 |
+
Step 17050/50462 | Loss: 2.6338 | LR: 3.79e-04 | 5252.9ms/step | 11.4 samples/s
|
| 666 |
+
Step 17060/50462 | Loss: 2.6589 | LR: 3.79e-04 | 5249.8ms/step | 11.4 samples/s
|
| 667 |
+
Step 17070/50462 | Loss: 2.6402 | LR: 3.78e-04 | 5255.6ms/step | 11.4 samples/s
|
| 668 |
+
Step 17080/50462 | Loss: 2.6374 | LR: 3.78e-04 | 5253.7ms/step | 11.4 samples/s
|
| 669 |
+
Step 17090/50462 | Loss: 2.6449 | LR: 3.78e-04 | 5255.4ms/step | 11.4 samples/s
|
| 670 |
+
Step 17100/50462 | Loss: 2.6580 | LR: 3.78e-04 | 5250.7ms/step | 11.4 samples/s
|
| 671 |
+
Step 17110/50462 | Loss: 2.6451 | LR: 3.78e-04 | 5251.8ms/step | 11.4 samples/s
|
| 672 |
+
Step 17120/50462 | Loss: 2.6504 | LR: 3.78e-04 | 5246.6ms/step | 11.4 samples/s
|
| 673 |
+
Step 17130/50462 | Loss: 2.6430 | LR: 3.78e-04 | 5253.6ms/step | 11.4 samples/s
|
| 674 |
+
Step 17140/50462 | Loss: 2.6290 | LR: 3.77e-04 | 5247.7ms/step | 11.4 samples/s
|
| 675 |
+
Step 17150/50462 | Loss: 2.6552 | LR: 3.77e-04 | 5248.3ms/step | 11.4 samples/s
|
| 676 |
+
Step 17160/50462 | Loss: 2.6506 | LR: 3.77e-04 | 5249.6ms/step | 11.4 samples/s
|
| 677 |
+
Step 17170/50462 | Loss: 2.6390 | LR: 3.77e-04 | 5253.0ms/step | 11.4 samples/s
|
| 678 |
+
Step 17180/50462 | Loss: 2.6875 | LR: 3.77e-04 | 5250.3ms/step | 11.4 samples/s
|
| 679 |
+
Step 17190/50462 | Loss: 2.6423 | LR: 3.77e-04 | 5249.1ms/step | 11.4 samples/s
|
| 680 |
+
Step 17200/50462 | Loss: 2.6407 | LR: 3.77e-04 | 5250.5ms/step | 11.4 samples/s
|
| 681 |
+
Step 17210/50462 | Loss: 2.6438 | LR: 3.77e-04 | 5248.3ms/step | 11.4 samples/s
|
| 682 |
+
Step 17220/50462 | Loss: 2.6455 | LR: 3.76e-04 | 5248.4ms/step | 11.4 samples/s
|
| 683 |
+
Step 17230/50462 | Loss: 2.6607 | LR: 3.76e-04 | 5253.3ms/step | 11.4 samples/s
|
| 684 |
+
Step 17240/50462 | Loss: 2.6410 | LR: 3.76e-04 | 5248.1ms/step | 11.4 samples/s
|
| 685 |
+
Step 17250/50462 | Loss: 2.6437 | LR: 3.76e-04 | 5252.3ms/step | 11.4 samples/s
|
| 686 |
+
Validation at step 17250: Loss = 2.6628, Perplexity = 14.34
|
| 687 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17250
|
| 688 |
+
Step 17260/50462 | Loss: 2.6405 | LR: 3.76e-04 | 10173.7ms/step | 5.9 samples/s
|
| 689 |
+
Step 17270/50462 | Loss: 2.6494 | LR: 3.76e-04 | 5250.0ms/step | 11.4 samples/s
|
| 690 |
+
Step 17280/50462 | Loss: 2.6384 | LR: 3.76e-04 | 5253.0ms/step | 11.4 samples/s
|
| 691 |
+
Step 17290/50462 | Loss: 2.6252 | LR: 3.75e-04 | 5252.2ms/step | 11.4 samples/s
|
| 692 |
+
Step 17300/50462 | Loss: 2.6299 | LR: 3.75e-04 | 5253.6ms/step | 11.4 samples/s
|
| 693 |
+
Step 17310/50462 | Loss: 2.6536 | LR: 3.75e-04 | 5253.3ms/step | 11.4 samples/s
|
| 694 |
+
Step 17320/50462 | Loss: 2.6200 | LR: 3.75e-04 | 5256.8ms/step | 11.4 samples/s
|
| 695 |
+
Step 17330/50462 | Loss: 2.6409 | LR: 3.75e-04 | 5254.5ms/step | 11.4 samples/s
|
| 696 |
+
Step 17340/50462 | Loss: 2.6635 | LR: 3.75e-04 | 5259.4ms/step | 11.4 samples/s
|
| 697 |
+
Step 17350/50462 | Loss: 2.6307 | LR: 3.75e-04 | 5253.3ms/step | 11.4 samples/s
|
| 698 |
+
Step 17360/50462 | Loss: 2.6770 | LR: 3.74e-04 | 5251.1ms/step | 11.4 samples/s
|
| 699 |
+
Step 17370/50462 | Loss: 2.6522 | LR: 3.74e-04 | 5245.6ms/step | 11.4 samples/s
|
| 700 |
+
Step 17380/50462 | Loss: 2.6659 | LR: 3.74e-04 | 5249.0ms/step | 11.4 samples/s
|
| 701 |
+
Step 17390/50462 | Loss: 2.6695 | LR: 3.74e-04 | 5250.2ms/step | 11.4 samples/s
|
| 702 |
+
Step 17400/50462 | Loss: 2.6566 | LR: 3.74e-04 | 5255.3ms/step | 11.4 samples/s
|
| 703 |
+
Step 17410/50462 | Loss: 2.6340 | LR: 3.74e-04 | 5252.1ms/step | 11.4 samples/s
|
| 704 |
+
Step 17420/50462 | Loss: 2.6584 | LR: 3.74e-04 | 5252.8ms/step | 11.4 samples/s
|
| 705 |
+
Step 17430/50462 | Loss: 2.6131 | LR: 3.74e-04 | 5255.4ms/step | 11.4 samples/s
|
| 706 |
+
Step 17440/50462 | Loss: 2.6714 | LR: 3.73e-04 | 5254.2ms/step | 11.4 samples/s
|
| 707 |
+
Step 17450/50462 | Loss: 2.6423 | LR: 3.73e-04 | 5252.5ms/step | 11.4 samples/s
|
| 708 |
+
Step 17460/50462 | Loss: 2.6522 | LR: 3.73e-04 | 5248.4ms/step | 11.4 samples/s
|
| 709 |
+
Step 17470/50462 | Loss: 2.6580 | LR: 3.73e-04 | 5252.1ms/step | 11.4 samples/s
|
| 710 |
+
Step 17480/50462 | Loss: 2.6617 | LR: 3.73e-04 | 5252.4ms/step | 11.4 samples/s
|
| 711 |
+
Step 17490/50462 | Loss: 2.6718 | LR: 3.73e-04 | 5253.1ms/step | 11.4 samples/s
|
| 712 |
+
Step 17500/50462 | Loss: 2.6603 | LR: 3.73e-04 | 5258.6ms/step | 11.4 samples/s
|
| 713 |
+
Validation at step 17500: Loss = 2.6592, Perplexity = 14.28
|
| 714 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17500
|
| 715 |
+
Step 17510/50462 | Loss: 2.6720 | LR: 3.72e-04 | 10142.9ms/step | 5.9 samples/s
|
| 716 |
+
Step 17520/50462 | Loss: 2.6499 | LR: 3.72e-04 | 5249.2ms/step | 11.4 samples/s
|
| 717 |
+
Step 17530/50462 | Loss: 2.6210 | LR: 3.72e-04 | 5251.1ms/step | 11.4 samples/s
|
| 718 |
+
Step 17540/50462 | Loss: 2.6730 | LR: 3.72e-04 | 5250.4ms/step | 11.4 samples/s
|
| 719 |
+
Step 17550/50462 | Loss: 2.6690 | LR: 3.72e-04 | 5249.9ms/step | 11.4 samples/s
|
| 720 |
+
Step 17560/50462 | Loss: 2.6138 | LR: 3.72e-04 | 5252.5ms/step | 11.4 samples/s
|
| 721 |
+
Step 17570/50462 | Loss: 2.6407 | LR: 3.72e-04 | 5250.2ms/step | 11.4 samples/s
|
| 722 |
+
Step 17580/50462 | Loss: 2.6359 | LR: 3.71e-04 | 5252.8ms/step | 11.4 samples/s
|
| 723 |
+
Step 17590/50462 | Loss: 2.6282 | LR: 3.71e-04 | 5248.8ms/step | 11.4 samples/s
|
| 724 |
+
Step 17600/50462 | Loss: 2.6619 | LR: 3.71e-04 | 5250.8ms/step | 11.4 samples/s
|
| 725 |
+
Step 17610/50462 | Loss: 2.6151 | LR: 3.71e-04 | 5257.4ms/step | 11.4 samples/s
|
| 726 |
+
Step 17620/50462 | Loss: 2.6598 | LR: 3.71e-04 | 5250.4ms/step | 11.4 samples/s
|
| 727 |
+
Step 17630/50462 | Loss: 2.6456 | LR: 3.71e-04 | 5250.6ms/step | 11.4 samples/s
|
| 728 |
+
Step 17640/50462 | Loss: 2.6312 | LR: 3.71e-04 | 5249.1ms/step | 11.4 samples/s
|
| 729 |
+
Step 17650/50462 | Loss: 2.6589 | LR: 3.70e-04 | 5253.0ms/step | 11.4 samples/s
|
| 730 |
+
Step 17660/50462 | Loss: 2.6411 | LR: 3.70e-04 | 5250.6ms/step | 11.4 samples/s
|
| 731 |
+
Step 17670/50462 | Loss: 2.6482 | LR: 3.70e-04 | 5256.3ms/step | 11.4 samples/s
|
| 732 |
+
Step 17680/50462 | Loss: 2.6723 | LR: 3.70e-04 | 5250.4ms/step | 11.4 samples/s
|
| 733 |
+
Step 17690/50462 | Loss: 2.6560 | LR: 3.70e-04 | 5251.1ms/step | 11.4 samples/s
|
| 734 |
+
Step 17700/50462 | Loss: 2.6382 | LR: 3.70e-04 | 5248.1ms/step | 11.4 samples/s
|
| 735 |
+
Step 17710/50462 | Loss: 2.6354 | LR: 3.70e-04 | 5254.4ms/step | 11.4 samples/s
|
| 736 |
+
Step 17720/50462 | Loss: 2.6418 | LR: 3.69e-04 | 5253.3ms/step | 11.4 samples/s
|
| 737 |
+
Step 17730/50462 | Loss: 2.6437 | LR: 3.69e-04 | 5254.1ms/step | 11.4 samples/s
|
| 738 |
+
Step 17740/50462 | Loss: 2.6566 | LR: 3.69e-04 | 5250.2ms/step | 11.4 samples/s
|
| 739 |
+
Step 17750/50462 | Loss: 2.6687 | LR: 3.69e-04 | 5252.4ms/step | 11.4 samples/s
|
| 740 |
+
Validation at step 17750: Loss = 2.6547, Perplexity = 14.22
|
| 741 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-17750
|
| 742 |
+
Step 17760/50462 | Loss: 2.6500 | LR: 3.69e-04 | 10154.7ms/step | 5.9 samples/s
|
| 743 |
+
Step 17770/50462 | Loss: 2.6273 | LR: 3.69e-04 | 5228.6ms/step | 11.5 samples/s
|
| 744 |
+
Step 17780/50462 | Loss: 2.6318 | LR: 3.69e-04 | 5230.5ms/step | 11.5 samples/s
|
| 745 |
+
Step 17790/50462 | Loss: 2.6299 | LR: 3.69e-04 | 5232.2ms/step | 11.5 samples/s
|
| 746 |
+
Step 17800/50462 | Loss: 2.6275 | LR: 3.68e-04 | 5230.8ms/step | 11.5 samples/s
|
| 747 |
+
Step 17810/50462 | Loss: 2.6502 | LR: 3.68e-04 | 5229.2ms/step | 11.5 samples/s
|
| 748 |
+
Step 17820/50462 | Loss: 2.6246 | LR: 3.68e-04 | 5230.9ms/step | 11.5 samples/s
|
| 749 |
+
Step 17830/50462 | Loss: 2.6453 | LR: 3.68e-04 | 5234.3ms/step | 11.5 samples/s
|
| 750 |
+
Step 17840/50462 | Loss: 2.6452 | LR: 3.68e-04 | 5222.7ms/step | 11.5 samples/s
|
| 751 |
+
Step 17850/50462 | Loss: 2.6334 | LR: 3.68e-04 | 5230.4ms/step | 11.5 samples/s
|
| 752 |
+
Step 17860/50462 | Loss: 2.6423 | LR: 3.68e-04 | 5223.4ms/step | 11.5 samples/s
|
| 753 |
+
Step 17870/50462 | Loss: 2.6414 | LR: 3.67e-04 | 5228.9ms/step | 11.5 samples/s
|
| 754 |
+
Step 17880/50462 | Loss: 2.6315 | LR: 3.67e-04 | 5226.8ms/step | 11.5 samples/s
|
| 755 |
+
Step 17890/50462 | Loss: 2.6652 | LR: 3.67e-04 | 5224.9ms/step | 11.5 samples/s
|
| 756 |
+
Step 17900/50462 | Loss: 2.6311 | LR: 3.67e-04 | 5221.1ms/step | 11.5 samples/s
|
| 757 |
+
Step 17910/50462 | Loss: 2.6430 | LR: 3.67e-04 | 5223.9ms/step | 11.5 samples/s
|
| 758 |
+
Step 17920/50462 | Loss: 2.6547 | LR: 3.67e-04 | 5231.8ms/step | 11.5 samples/s
|
| 759 |
+
Step 17930/50462 | Loss: 2.6255 | LR: 3.67e-04 | 5225.5ms/step | 11.5 samples/s
|
| 760 |
+
Step 17940/50462 | Loss: 2.6618 | LR: 3.66e-04 | 5226.0ms/step | 11.5 samples/s
|
| 761 |
+
Step 17950/50462 | Loss: 2.6619 | LR: 3.66e-04 | 5228.3ms/step | 11.5 samples/s
|
| 762 |
+
Step 17960/50462 | Loss: 2.6457 | LR: 3.66e-04 | 5237.6ms/step | 11.5 samples/s
|
| 763 |
+
Step 17970/50462 | Loss: 2.5900 | LR: 3.66e-04 | 5225.7ms/step | 11.5 samples/s
|
| 764 |
+
Step 17980/50462 | Loss: 2.6449 | LR: 3.66e-04 | 5227.1ms/step | 11.5 samples/s
|
| 765 |
+
Step 17990/50462 | Loss: 2.6251 | LR: 3.66e-04 | 5230.2ms/step | 11.5 samples/s
|
| 766 |
+
Step 18000/50462 | Loss: 2.6323 | LR: 3.66e-04 | 5229.0ms/step | 11.5 samples/s
|
| 767 |
+
Validation at step 18000: Loss = 2.6514, Perplexity = 14.17
|
| 768 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18000
|
| 769 |
+
Step 18010/50462 | Loss: 2.6503 | LR: 3.65e-04 | 10111.0ms/step | 5.9 samples/s
|
| 770 |
+
Step 18020/50462 | Loss: 2.6422 | LR: 3.65e-04 | 5251.2ms/step | 11.4 samples/s
|
| 771 |
+
Step 18030/50462 | Loss: 2.6458 | LR: 3.65e-04 | 5255.8ms/step | 11.4 samples/s
|
| 772 |
+
Step 18040/50462 | Loss: 2.6288 | LR: 3.65e-04 | 5246.3ms/step | 11.4 samples/s
|
| 773 |
+
Step 18050/50462 | Loss: 2.6522 | LR: 3.65e-04 | 5250.1ms/step | 11.4 samples/s
|
| 774 |
+
Step 18060/50462 | Loss: 2.6368 | LR: 3.65e-04 | 5253.2ms/step | 11.4 samples/s
|
| 775 |
+
Step 18070/50462 | Loss: 2.6324 | LR: 3.65e-04 | 5254.7ms/step | 11.4 samples/s
|
| 776 |
+
Step 18080/50462 | Loss: 2.6278 | LR: 3.64e-04 | 5248.6ms/step | 11.4 samples/s
|
| 777 |
+
Step 18090/50462 | Loss: 2.6333 | LR: 3.64e-04 | 5252.0ms/step | 11.4 samples/s
|
| 778 |
+
Step 18100/50462 | Loss: 2.6510 | LR: 3.64e-04 | 5252.4ms/step | 11.4 samples/s
|
| 779 |
+
Step 18110/50462 | Loss: 2.6447 | LR: 3.64e-04 | 5254.2ms/step | 11.4 samples/s
|
| 780 |
+
Step 18120/50462 | Loss: 2.6260 | LR: 3.64e-04 | 5255.3ms/step | 11.4 samples/s
|
| 781 |
+
Step 18130/50462 | Loss: 2.6412 | LR: 3.64e-04 | 5250.4ms/step | 11.4 samples/s
|
| 782 |
+
Step 18140/50462 | Loss: 2.6437 | LR: 3.64e-04 | 5252.8ms/step | 11.4 samples/s
|
| 783 |
+
Step 18150/50462 | Loss: 2.6419 | LR: 3.63e-04 | 5255.1ms/step | 11.4 samples/s
|
| 784 |
+
Step 18160/50462 | Loss: 2.6495 | LR: 3.63e-04 | 5247.5ms/step | 11.4 samples/s
|
| 785 |
+
Step 18170/50462 | Loss: 2.6320 | LR: 3.63e-04 | 5249.9ms/step | 11.4 samples/s
|
| 786 |
+
Step 18180/50462 | Loss: 2.6394 | LR: 3.63e-04 | 5254.1ms/step | 11.4 samples/s
|
| 787 |
+
Step 18190/50462 | Loss: 2.6502 | LR: 3.63e-04 | 5251.5ms/step | 11.4 samples/s
|
| 788 |
+
Step 18200/50462 | Loss: 2.6213 | LR: 3.63e-04 | 5250.4ms/step | 11.4 samples/s
|
| 789 |
+
Step 18210/50462 | Loss: 2.6139 | LR: 3.63e-04 | 5257.4ms/step | 11.4 samples/s
|
| 790 |
+
Step 18220/50462 | Loss: 2.6543 | LR: 3.62e-04 | 5242.3ms/step | 11.4 samples/s
|
| 791 |
+
Step 18230/50462 | Loss: 2.6328 | LR: 3.62e-04 | 5251.2ms/step | 11.4 samples/s
|
| 792 |
+
Step 18240/50462 | Loss: 2.6410 | LR: 3.62e-04 | 5252.4ms/step | 11.4 samples/s
|
| 793 |
+
Step 18250/50462 | Loss: 2.6347 | LR: 3.62e-04 | 5256.6ms/step | 11.4 samples/s
|
| 794 |
+
Validation at step 18250: Loss = 2.6481, Perplexity = 14.13
|
| 795 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18250
|
| 796 |
+
Step 18260/50462 | Loss: 2.6413 | LR: 3.62e-04 | 10092.8ms/step | 5.9 samples/s
|
| 797 |
+
Step 18270/50462 | Loss: 2.6195 | LR: 3.62e-04 | 5257.1ms/step | 11.4 samples/s
|
| 798 |
+
Step 18280/50462 | Loss: 2.6301 | LR: 3.62e-04 | 5260.2ms/step | 11.4 samples/s
|
| 799 |
+
Step 18290/50462 | Loss: 2.6350 | LR: 3.62e-04 | 5254.7ms/step | 11.4 samples/s
|
| 800 |
+
Step 18300/50462 | Loss: 2.6181 | LR: 3.61e-04 | 5247.3ms/step | 11.4 samples/s
|
| 801 |
+
Step 18310/50462 | Loss: 2.6135 | LR: 3.61e-04 | 5253.4ms/step | 11.4 samples/s
|
| 802 |
+
Step 18320/50462 | Loss: 2.6189 | LR: 3.61e-04 | 5251.8ms/step | 11.4 samples/s
|
| 803 |
+
Step 18330/50462 | Loss: 2.6410 | LR: 3.61e-04 | 5250.5ms/step | 11.4 samples/s
|
| 804 |
+
Step 18340/50462 | Loss: 2.6603 | LR: 3.61e-04 | 5250.3ms/step | 11.4 samples/s
|
| 805 |
+
Step 18350/50462 | Loss: 2.6313 | LR: 3.61e-04 | 5246.0ms/step | 11.4 samples/s
|
| 806 |
+
Step 18360/50462 | Loss: 2.6398 | LR: 3.61e-04 | 5249.1ms/step | 11.4 samples/s
|
| 807 |
+
Step 18370/50462 | Loss: 2.6141 | LR: 3.60e-04 | 5251.8ms/step | 11.4 samples/s
|
| 808 |
+
Step 18380/50462 | Loss: 2.6303 | LR: 3.60e-04 | 5249.6ms/step | 11.4 samples/s
|
| 809 |
+
Step 18390/50462 | Loss: 2.6404 | LR: 3.60e-04 | 5252.8ms/step | 11.4 samples/s
|
| 810 |
+
Step 18400/50462 | Loss: 2.6474 | LR: 3.60e-04 | 5251.3ms/step | 11.4 samples/s
|
| 811 |
+
Step 18410/50462 | Loss: 2.6348 | LR: 3.60e-04 | 5255.1ms/step | 11.4 samples/s
|
| 812 |
+
Step 18420/50462 | Loss: 2.6236 | LR: 3.60e-04 | 5249.8ms/step | 11.4 samples/s
|
| 813 |
+
Step 18430/50462 | Loss: 2.6730 | LR: 3.60e-04 | 5253.9ms/step | 11.4 samples/s
|
| 814 |
+
Step 18440/50462 | Loss: 2.6279 | LR: 3.59e-04 | 5248.2ms/step | 11.4 samples/s
|
| 815 |
+
Step 18450/50462 | Loss: 2.6321 | LR: 3.59e-04 | 5252.9ms/step | 11.4 samples/s
|
| 816 |
+
Step 18460/50462 | Loss: 2.6222 | LR: 3.59e-04 | 5250.5ms/step | 11.4 samples/s
|
| 817 |
+
Step 18470/50462 | Loss: 2.6298 | LR: 3.59e-04 | 5254.6ms/step | 11.4 samples/s
|
| 818 |
+
Step 18480/50462 | Loss: 2.6228 | LR: 3.59e-04 | 5250.4ms/step | 11.4 samples/s
|
| 819 |
+
Step 18490/50462 | Loss: 2.6252 | LR: 3.59e-04 | 5245.3ms/step | 11.4 samples/s
|
| 820 |
+
Step 18500/50462 | Loss: 2.6440 | LR: 3.59e-04 | 5249.4ms/step | 11.4 samples/s
|
| 821 |
+
Validation at step 18500: Loss = 2.6449, Perplexity = 14.08
|
| 822 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18500
|
| 823 |
+
Step 18510/50462 | Loss: 2.6599 | LR: 3.58e-04 | 10124.3ms/step | 5.9 samples/s
|
| 824 |
+
Step 18520/50462 | Loss: 2.6227 | LR: 3.58e-04 | 5230.6ms/step | 11.5 samples/s
|
| 825 |
+
Step 18530/50462 | Loss: 2.6243 | LR: 3.58e-04 | 5227.3ms/step | 11.5 samples/s
|
| 826 |
+
Step 18540/50462 | Loss: 2.6135 | LR: 3.58e-04 | 5228.1ms/step | 11.5 samples/s
|
| 827 |
+
Step 18550/50462 | Loss: 2.6253 | LR: 3.58e-04 | 5229.5ms/step | 11.5 samples/s
|
| 828 |
+
Step 18560/50462 | Loss: 2.6301 | LR: 3.58e-04 | 5232.2ms/step | 11.5 samples/s
|
| 829 |
+
Step 18570/50462 | Loss: 2.6390 | LR: 3.58e-04 | 5231.8ms/step | 11.5 samples/s
|
| 830 |
+
Step 18580/50462 | Loss: 2.6244 | LR: 3.57e-04 | 5223.8ms/step | 11.5 samples/s
|
| 831 |
+
Step 18590/50462 | Loss: 2.6267 | LR: 3.57e-04 | 5219.4ms/step | 11.5 samples/s
|
| 832 |
+
Step 18600/50462 | Loss: 2.6421 | LR: 3.57e-04 | 5238.2ms/step | 11.5 samples/s
|
| 833 |
+
Step 18610/50462 | Loss: 2.6310 | LR: 3.57e-04 | 5253.0ms/step | 11.4 samples/s
|
| 834 |
+
Step 18620/50462 | Loss: 2.6210 | LR: 3.57e-04 | 5247.2ms/step | 11.4 samples/s
|
| 835 |
+
Step 18630/50462 | Loss: 2.6248 | LR: 3.57e-04 | 5254.0ms/step | 11.4 samples/s
|
| 836 |
+
Step 18640/50462 | Loss: 2.6173 | LR: 3.57e-04 | 5249.4ms/step | 11.4 samples/s
|
| 837 |
+
Step 18650/50462 | Loss: 2.6290 | LR: 3.56e-04 | 5253.7ms/step | 11.4 samples/s
|
| 838 |
+
Step 18660/50462 | Loss: 2.6287 | LR: 3.56e-04 | 5246.8ms/step | 11.4 samples/s
|
| 839 |
+
Step 18670/50462 | Loss: 2.6159 | LR: 3.56e-04 | 5252.0ms/step | 11.4 samples/s
|
| 840 |
+
Step 18680/50462 | Loss: 2.6228 | LR: 3.56e-04 | 5257.6ms/step | 11.4 samples/s
|
| 841 |
+
Step 18690/50462 | Loss: 2.6265 | LR: 3.56e-04 | 5247.9ms/step | 11.4 samples/s
|
| 842 |
+
Step 18700/50462 | Loss: 2.6232 | LR: 3.56e-04 | 5254.7ms/step | 11.4 samples/s
|
| 843 |
+
Step 18710/50462 | Loss: 2.6366 | LR: 3.56e-04 | 5244.9ms/step | 11.4 samples/s
|
| 844 |
+
Step 18720/50462 | Loss: 2.6347 | LR: 3.55e-04 | 5248.5ms/step | 11.4 samples/s
|
| 845 |
+
Step 18730/50462 | Loss: 2.6132 | LR: 3.55e-04 | 5254.3ms/step | 11.4 samples/s
|
| 846 |
+
Step 18740/50462 | Loss: 2.6292 | LR: 3.55e-04 | 5257.1ms/step | 11.4 samples/s
|
| 847 |
+
Step 18750/50462 | Loss: 2.6268 | LR: 3.55e-04 | 5254.7ms/step | 11.4 samples/s
|
| 848 |
+
Validation at step 18750: Loss = 2.6417, Perplexity = 14.04
|
| 849 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-18750
|
| 850 |
+
Step 18760/50462 | Loss: 2.6438 | LR: 3.55e-04 | 10176.4ms/step | 5.9 samples/s
|
| 851 |
+
Step 18770/50462 | Loss: 2.6242 | LR: 3.55e-04 | 5252.1ms/step | 11.4 samples/s
|
| 852 |
+
Step 18780/50462 | Loss: 2.6241 | LR: 3.55e-04 | 5250.9ms/step | 11.4 samples/s
|
| 853 |
+
Step 18790/50462 | Loss: 2.6496 | LR: 3.54e-04 | 5248.4ms/step | 11.4 samples/s
|
| 854 |
+
Step 18800/50462 | Loss: 2.6185 | LR: 3.54e-04 | 5245.6ms/step | 11.4 samples/s
|
| 855 |
+
Step 18810/50462 | Loss: 2.6404 | LR: 3.54e-04 | 5252.5ms/step | 11.4 samples/s
|
| 856 |
+
Step 18820/50462 | Loss: 2.6365 | LR: 3.54e-04 | 5247.5ms/step | 11.4 samples/s
|
| 857 |
+
Step 18830/50462 | Loss: 2.6339 | LR: 3.54e-04 | 5249.8ms/step | 11.4 samples/s
|
| 858 |
+
Step 18840/50462 | Loss: 2.6336 | LR: 3.54e-04 | 5247.8ms/step | 11.4 samples/s
|
| 859 |
+
Step 18850/50462 | Loss: 2.6368 | LR: 3.54e-04 | 5250.2ms/step | 11.4 samples/s
|
| 860 |
+
Step 18860/50462 | Loss: 2.6482 | LR: 3.53e-04 | 5251.5ms/step | 11.4 samples/s
|
| 861 |
+
Step 18870/50462 | Loss: 2.5955 | LR: 3.53e-04 | 5250.2ms/step | 11.4 samples/s
|
| 862 |
+
Step 18880/50462 | Loss: 2.6281 | LR: 3.53e-04 | 5250.6ms/step | 11.4 samples/s
|
| 863 |
+
Step 18890/50462 | Loss: 2.6123 | LR: 3.53e-04 | 5245.5ms/step | 11.4 samples/s
|
| 864 |
+
Step 18900/50462 | Loss: 2.6470 | LR: 3.53e-04 | 5255.0ms/step | 11.4 samples/s
|
| 865 |
+
Step 18910/50462 | Loss: 2.6380 | LR: 3.53e-04 | 5254.9ms/step | 11.4 samples/s
|
| 866 |
+
Step 18920/50462 | Loss: 2.6131 | LR: 3.53e-04 | 5251.3ms/step | 11.4 samples/s
|
| 867 |
+
Step 18930/50462 | Loss: 2.6396 | LR: 3.52e-04 | 5244.7ms/step | 11.4 samples/s
|
| 868 |
+
Step 18940/50462 | Loss: 2.6569 | LR: 3.52e-04 | 5247.4ms/step | 11.4 samples/s
|
| 869 |
+
Step 18950/50462 | Loss: 2.6368 | LR: 3.52e-04 | 5246.1ms/step | 11.4 samples/s
|
| 870 |
+
Step 18960/50462 | Loss: 2.6420 | LR: 3.52e-04 | 5253.2ms/step | 11.4 samples/s
|
| 871 |
+
Step 18970/50462 | Loss: 2.6348 | LR: 3.52e-04 | 5251.9ms/step | 11.4 samples/s
|
| 872 |
+
Step 18980/50462 | Loss: 2.6316 | LR: 3.52e-04 | 5248.7ms/step | 11.4 samples/s
|
| 873 |
+
Step 18990/50462 | Loss: 2.6176 | LR: 3.52e-04 | 5255.9ms/step | 11.4 samples/s
|
| 874 |
+
Step 19000/50462 | Loss: 2.6013 | LR: 3.51e-04 | 5249.3ms/step | 11.4 samples/s
|
| 875 |
+
Validation at step 19000: Loss = 2.6393, Perplexity = 14.00
|
| 876 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000
|
| 877 |
+
Step 19010/50462 | Loss: 2.6277 | LR: 3.51e-04 | 10113.9ms/step | 5.9 samples/s
|
| 878 |
+
Step 19020/50462 | Loss: 2.6279 | LR: 3.51e-04 | 5242.6ms/step | 11.4 samples/s
|
| 879 |
+
Step 19030/50462 | Loss: 2.6434 | LR: 3.51e-04 | 5249.5ms/step | 11.4 samples/s
|
| 880 |
+
Step 19040/50462 | Loss: 2.6177 | LR: 3.51e-04 | 5240.9ms/step | 11.4 samples/s
|
| 881 |
+
Step 19050/50462 | Loss: 2.6221 | LR: 3.51e-04 | 5229.6ms/step | 11.5 samples/s
|
| 882 |
+
Step 19060/50462 | Loss: 2.6198 | LR: 3.50e-04 | 5228.0ms/step | 11.5 samples/s
|
| 883 |
+
Step 19070/50462 | Loss: 2.6231 | LR: 3.50e-04 | 5244.7ms/step | 11.4 samples/s
|
| 884 |
+
Step 19080/50462 | Loss: 2.5970 | LR: 3.50e-04 | 5225.9ms/step | 11.5 samples/s
|
| 885 |
+
Step 19090/50462 | Loss: 2.6259 | LR: 3.50e-04 | 5228.2ms/step | 11.5 samples/s
|
| 886 |
+
Step 19100/50462 | Loss: 2.6249 | LR: 3.50e-04 | 5229.4ms/step | 11.5 samples/s
|
| 887 |
+
Step 19110/50462 | Loss: 2.6293 | LR: 3.50e-04 | 5238.8ms/step | 11.5 samples/s
|
| 888 |
+
Step 19120/50462 | Loss: 2.6366 | LR: 3.50e-04 | 5235.9ms/step | 11.5 samples/s
|
| 889 |
+
Step 19130/50462 | Loss: 2.6336 | LR: 3.49e-04 | 5228.9ms/step | 11.5 samples/s
|
| 890 |
+
Step 19140/50462 | Loss: 2.6194 | LR: 3.49e-04 | 5230.3ms/step | 11.5 samples/s
|
| 891 |
+
Step 19150/50462 | Loss: 2.6282 | LR: 3.49e-04 | 5228.5ms/step | 11.5 samples/s
|
| 892 |
+
Step 19160/50462 | Loss: 2.6331 | LR: 3.49e-04 | 5228.4ms/step | 11.5 samples/s
|
| 893 |
+
Step 19170/50462 | Loss: 2.6337 | LR: 3.49e-04 | 5235.1ms/step | 11.5 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume2.log
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0825 03:17:03.225000 152 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0825 03:17:03.225000 152 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0825 03:17:03.225000 152 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0825 03:17:03.225000 152 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 2,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 2gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
================================================================================
|
| 75 |
+
MODEL ARCHITECTURE
|
| 76 |
+
================================================================================
|
| 77 |
+
Model config:
|
| 78 |
+
vocab_size: 151936
|
| 79 |
+
hidden_size: 1536
|
| 80 |
+
num_hidden_layers: 28
|
| 81 |
+
num_attention_heads: 12
|
| 82 |
+
num_key_value_heads: 4
|
| 83 |
+
head_dim: 128
|
| 84 |
+
intermediate_size: 6144
|
| 85 |
+
max_position_embeddings: 32768
|
| 86 |
+
rope_theta: 1000000
|
| 87 |
+
rope_train_position_scale_set: None
|
| 88 |
+
attention_type: sparse_stochastic
|
| 89 |
+
sparse_block_size: 128
|
| 90 |
+
sparse_num_random_blocks: 31
|
| 91 |
+
sparse_mask_pool_size: 1
|
| 92 |
+
sparse_max_seq_length: 8192
|
| 93 |
+
|
| 94 |
+
Parameter count:
|
| 95 |
+
Total parameters: 1,202,352,640
|
| 96 |
+
Trainable parameters: 1,202,352,640
|
| 97 |
+
Non-trainable parameters: 0
|
| 98 |
+
Model size (BF16): 2.40 GB
|
| 99 |
+
|
| 100 |
+
--------------------------------------------------------------------------------
|
| 101 |
+
LAYER-BY-LAYER SUMMARY
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
================================================================================================================================================================
|
| 104 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 107 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True
|
| 108 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True
|
| 109 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True
|
| 139 |
+
├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True
|
| 140 |
+
================================================================================================================================================================
|
| 141 |
+
Total params: 1,435,726,336
|
| 142 |
+
Trainable params: 1,435,726,336
|
| 143 |
+
Non-trainable params: 0
|
| 144 |
+
Total mult-adds (Units.GIGABYTES): 2.87
|
| 145 |
+
================================================================================================================================================================
|
| 146 |
+
Input size (MB): 0.13
|
| 147 |
+
Forward/backward pass size (MB): 52437.19
|
| 148 |
+
Params size (MB): 2871.45
|
| 149 |
+
Estimated Total Size (MB): 55308.77
|
| 150 |
+
================================================================================================================================================================
|
| 151 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 152 |
+
================================================================================
|
| 153 |
+
|
| 154 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 155 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 156 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 159 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 160 |
+
Dataset split: 3027764 train, 159356 validation
|
| 161 |
+
Training examples (packed, train split): 3027764
|
| 162 |
+
Validation examples (packed, val split): 159356
|
| 163 |
+
Per-device batch size: 2
|
| 164 |
+
World size (GPUs): 2
|
| 165 |
+
Gradient accumulation steps: 15
|
| 166 |
+
Effective batch size (seqs/step): 60
|
| 167 |
+
Sequence length: 8192
|
| 168 |
+
Tokens per optimizer step: 491,520
|
| 169 |
+
Micro-batches per epoch (per rank): 756941
|
| 170 |
+
Optimizer steps per epoch: 50462
|
| 171 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 172 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19000...
|
| 173 |
+
Preserving checkpoint LR state for exact resume: current=[0.0003513626837771881], schedule_base=[0.0005]
|
| 174 |
+
Resumed from checkpoint at step 19000 (epoch 0, micro_step 258735)
|
| 175 |
+
Resuming training from step 19000
|
| 176 |
+
Resuming wandb run rgmkfpnl
|
| 177 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 178 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 179 |
+
wandb: setting up run rgmkfpnl
|
| 180 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 181 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_031836-rgmkfpnl
|
| 182 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 183 |
+
wandb: Resuming run 1.2b dense 8k seed70 2gpu resume
|
| 184 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 185 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 186 |
+
Starting training...
|
| 187 |
+
Validation at step 19000: Loss = 2.6393, Perplexity = 14.00
|
| 188 |
+
wandb: WARNING Tried to log to step 19000 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 189 |
+
Switching attention mode for step 19001: dense (dense step interval)
|
| 190 |
+
Step 19010/50462 | Loss: 2.6276 | LR: 3.51e-04 | 58825.0ms/step | 1.0 samples/s
|
| 191 |
+
wandb: WARNING Tried to log to step 19010 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 192 |
+
Step 19020/50462 | Loss: 2.6279 | LR: 3.51e-04 | 5183.6ms/step | 11.6 samples/s
|
| 193 |
+
wandb: WARNING Tried to log to step 19020 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 194 |
+
Step 19030/50462 | Loss: 2.6434 | LR: 3.51e-04 | 5177.4ms/step | 11.6 samples/s
|
| 195 |
+
wandb: WARNING Tried to log to step 19030 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 196 |
+
Step 19040/50462 | Loss: 2.6177 | LR: 3.51e-04 | 5186.6ms/step | 11.6 samples/s
|
| 197 |
+
wandb: WARNING Tried to log to step 19040 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 198 |
+
Step 19050/50462 | Loss: 2.6221 | LR: 3.51e-04 | 5186.3ms/step | 11.6 samples/s
|
| 199 |
+
wandb: WARNING Tried to log to step 19050 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Step 19060/50462 | Loss: 2.6198 | LR: 3.50e-04 | 5184.0ms/step | 11.6 samples/s
|
| 201 |
+
wandb: WARNING Tried to log to step 19060 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 202 |
+
Step 19070/50462 | Loss: 2.6230 | LR: 3.50e-04 | 5177.5ms/step | 11.6 samples/s
|
| 203 |
+
wandb: WARNING Tried to log to step 19070 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 204 |
+
Step 19080/50462 | Loss: 2.5970 | LR: 3.50e-04 | 5177.9ms/step | 11.6 samples/s
|
| 205 |
+
wandb: WARNING Tried to log to step 19080 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 206 |
+
Step 19090/50462 | Loss: 2.6260 | LR: 3.50e-04 | 5197.3ms/step | 11.5 samples/s
|
| 207 |
+
wandb: WARNING Tried to log to step 19090 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 208 |
+
Step 19100/50462 | Loss: 2.6249 | LR: 3.50e-04 | 5206.8ms/step | 11.5 samples/s
|
| 209 |
+
wandb: WARNING Tried to log to step 19100 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Step 19110/50462 | Loss: 2.6293 | LR: 3.50e-04 | 5206.9ms/step | 11.5 samples/s
|
| 211 |
+
wandb: WARNING Tried to log to step 19110 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 212 |
+
Step 19120/50462 | Loss: 2.6367 | LR: 3.50e-04 | 5205.9ms/step | 11.5 samples/s
|
| 213 |
+
wandb: WARNING Tried to log to step 19120 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Step 19130/50462 | Loss: 2.6336 | LR: 3.49e-04 | 5207.3ms/step | 11.5 samples/s
|
| 215 |
+
wandb: WARNING Tried to log to step 19130 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 216 |
+
Step 19140/50462 | Loss: 2.6194 | LR: 3.49e-04 | 5209.7ms/step | 11.5 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 19140 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 19150/50462 | Loss: 2.6282 | LR: 3.49e-04 | 5211.6ms/step | 11.5 samples/s
|
| 219 |
+
wandb: WARNING Tried to log to step 19150 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 220 |
+
Step 19160/50462 | Loss: 2.6331 | LR: 3.49e-04 | 5213.4ms/step | 11.5 samples/s
|
| 221 |
+
wandb: WARNING Tried to log to step 19160 that is less than the current step 19161. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 222 |
+
Step 19170/50462 | Loss: 2.6337 | LR: 3.49e-04 | 5207.5ms/step | 11.5 samples/s
|
| 223 |
+
Step 19180/50462 | Loss: 2.6268 | LR: 3.49e-04 | 5205.1ms/step | 11.5 samples/s
|
| 224 |
+
Step 19190/50462 | Loss: 2.6396 | LR: 3.49e-04 | 5209.5ms/step | 11.5 samples/s
|
| 225 |
+
Step 19200/50462 | Loss: 2.6277 | LR: 3.48e-04 | 5215.3ms/step | 11.5 samples/s
|
| 226 |
+
Step 19210/50462 | Loss: 2.6454 | LR: 3.48e-04 | 5209.6ms/step | 11.5 samples/s
|
| 227 |
+
Step 19220/50462 | Loss: 2.6127 | LR: 3.48e-04 | 5199.9ms/step | 11.5 samples/s
|
| 228 |
+
Step 19230/50462 | Loss: 2.6041 | LR: 3.48e-04 | 5211.5ms/step | 11.5 samples/s
|
| 229 |
+
Step 19240/50462 | Loss: 2.6336 | LR: 3.48e-04 | 5210.4ms/step | 11.5 samples/s
|
| 230 |
+
Step 19250/50462 | Loss: 2.6102 | LR: 3.48e-04 | 5209.3ms/step | 11.5 samples/s
|
| 231 |
+
Validation at step 19250: Loss = 2.6360, Perplexity = 13.96
|
| 232 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19250
|
| 233 |
+
Step 19260/50462 | Loss: 2.6208 | LR: 3.48e-04 | 10086.1ms/step | 5.9 samples/s
|
| 234 |
+
Step 19270/50462 | Loss: 2.6216 | LR: 3.47e-04 | 5217.0ms/step | 11.5 samples/s
|
| 235 |
+
Step 19280/50462 | Loss: 2.6105 | LR: 3.47e-04 | 5209.9ms/step | 11.5 samples/s
|
| 236 |
+
Step 19290/50462 | Loss: 2.6180 | LR: 3.47e-04 | 5209.0ms/step | 11.5 samples/s
|
| 237 |
+
Step 19300/50462 | Loss: 2.6167 | LR: 3.47e-04 | 5206.7ms/step | 11.5 samples/s
|
| 238 |
+
Step 19310/50462 | Loss: 2.6291 | LR: 3.47e-04 | 5211.8ms/step | 11.5 samples/s
|
| 239 |
+
Step 19320/50462 | Loss: 2.6249 | LR: 3.47e-04 | 5213.9ms/step | 11.5 samples/s
|
| 240 |
+
Step 19330/50462 | Loss: 2.6130 | LR: 3.47e-04 | 5209.8ms/step | 11.5 samples/s
|
| 241 |
+
Step 19340/50462 | Loss: 2.6182 | LR: 3.46e-04 | 5217.0ms/step | 11.5 samples/s
|
| 242 |
+
Step 19350/50462 | Loss: 2.6240 | LR: 3.46e-04 | 5212.1ms/step | 11.5 samples/s
|
| 243 |
+
Step 19360/50462 | Loss: 2.6236 | LR: 3.46e-04 | 5212.8ms/step | 11.5 samples/s
|
| 244 |
+
Step 19370/50462 | Loss: 2.6154 | LR: 3.46e-04 | 5215.4ms/step | 11.5 samples/s
|
| 245 |
+
Step 19380/50462 | Loss: 2.5828 | LR: 3.46e-04 | 5209.3ms/step | 11.5 samples/s
|
| 246 |
+
Step 19390/50462 | Loss: 2.6033 | LR: 3.46e-04 | 5211.8ms/step | 11.5 samples/s
|
| 247 |
+
Step 19400/50462 | Loss: 2.6069 | LR: 3.46e-04 | 5213.9ms/step | 11.5 samples/s
|
| 248 |
+
Step 19410/50462 | Loss: 2.6311 | LR: 3.45e-04 | 5213.7ms/step | 11.5 samples/s
|
| 249 |
+
Step 19420/50462 | Loss: 2.6305 | LR: 3.45e-04 | 5215.1ms/step | 11.5 samples/s
|
| 250 |
+
Step 19430/50462 | Loss: 2.6104 | LR: 3.45e-04 | 5210.1ms/step | 11.5 samples/s
|
| 251 |
+
Step 19440/50462 | Loss: 2.6210 | LR: 3.45e-04 | 5213.0ms/step | 11.5 samples/s
|
| 252 |
+
Step 19450/50462 | Loss: 2.6450 | LR: 3.45e-04 | 5208.8ms/step | 11.5 samples/s
|
| 253 |
+
Step 19460/50462 | Loss: 2.5854 | LR: 3.45e-04 | 5212.8ms/step | 11.5 samples/s
|
| 254 |
+
Step 19470/50462 | Loss: 2.6128 | LR: 3.45e-04 | 5211.7ms/step | 11.5 samples/s
|
| 255 |
+
Step 19480/50462 | Loss: 2.6226 | LR: 3.44e-04 | 5216.4ms/step | 11.5 samples/s
|
| 256 |
+
Step 19490/50462 | Loss: 2.6375 | LR: 3.44e-04 | 5213.7ms/step | 11.5 samples/s
|
| 257 |
+
Step 19500/50462 | Loss: 2.6199 | LR: 3.44e-04 | 5210.8ms/step | 11.5 samples/s
|
| 258 |
+
Validation at step 19500: Loss = 2.6327, Perplexity = 13.91
|
| 259 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19500
|
| 260 |
+
Step 19510/50462 | Loss: 2.6441 | LR: 3.44e-04 | 10084.1ms/step | 5.9 samples/s
|
| 261 |
+
Step 19520/50462 | Loss: 2.6217 | LR: 3.44e-04 | 5188.9ms/step | 11.6 samples/s
|
| 262 |
+
Step 19530/50462 | Loss: 2.6357 | LR: 3.44e-04 | 5207.1ms/step | 11.5 samples/s
|
| 263 |
+
Step 19540/50462 | Loss: 2.6183 | LR: 3.44e-04 | 5215.2ms/step | 11.5 samples/s
|
| 264 |
+
Step 19550/50462 | Loss: 2.6490 | LR: 3.43e-04 | 5214.6ms/step | 11.5 samples/s
|
| 265 |
+
Step 19560/50462 | Loss: 2.6155 | LR: 3.43e-04 | 5209.2ms/step | 11.5 samples/s
|
| 266 |
+
Step 19570/50462 | Loss: 2.6192 | LR: 3.43e-04 | 5216.5ms/step | 11.5 samples/s
|
| 267 |
+
Step 19580/50462 | Loss: 2.6125 | LR: 3.43e-04 | 5218.2ms/step | 11.5 samples/s
|
| 268 |
+
Step 19590/50462 | Loss: 2.6193 | LR: 3.43e-04 | 5213.1ms/step | 11.5 samples/s
|
| 269 |
+
Step 19600/50462 | Loss: 2.6108 | LR: 3.43e-04 | 5209.8ms/step | 11.5 samples/s
|
| 270 |
+
Step 19610/50462 | Loss: 2.6287 | LR: 3.42e-04 | 5210.3ms/step | 11.5 samples/s
|
| 271 |
+
Step 19620/50462 | Loss: 2.6168 | LR: 3.42e-04 | 5210.2ms/step | 11.5 samples/s
|
| 272 |
+
Step 19630/50462 | Loss: 2.6554 | LR: 3.42e-04 | 5212.8ms/step | 11.5 samples/s
|
| 273 |
+
Step 19640/50462 | Loss: 2.6127 | LR: 3.42e-04 | 5210.0ms/step | 11.5 samples/s
|
| 274 |
+
Step 19650/50462 | Loss: 2.6324 | LR: 3.42e-04 | 5206.1ms/step | 11.5 samples/s
|
| 275 |
+
Step 19660/50462 | Loss: 2.6299 | LR: 3.42e-04 | 5211.5ms/step | 11.5 samples/s
|
| 276 |
+
Step 19670/50462 | Loss: 2.6102 | LR: 3.42e-04 | 5210.8ms/step | 11.5 samples/s
|
| 277 |
+
Step 19680/50462 | Loss: 2.5796 | LR: 3.41e-04 | 5210.2ms/step | 11.5 samples/s
|
| 278 |
+
Step 19690/50462 | Loss: 2.6139 | LR: 3.41e-04 | 5205.2ms/step | 11.5 samples/s
|
| 279 |
+
Step 19700/50462 | Loss: 2.6395 | LR: 3.41e-04 | 5212.5ms/step | 11.5 samples/s
|
| 280 |
+
Step 19710/50462 | Loss: 2.6274 | LR: 3.41e-04 | 5211.4ms/step | 11.5 samples/s
|
| 281 |
+
Step 19720/50462 | Loss: 2.6226 | LR: 3.41e-04 | 5214.9ms/step | 11.5 samples/s
|
| 282 |
+
Step 19730/50462 | Loss: 2.6307 | LR: 3.41e-04 | 5214.8ms/step | 11.5 samples/s
|
| 283 |
+
Step 19740/50462 | Loss: 2.6085 | LR: 3.41e-04 | 5216.6ms/step | 11.5 samples/s
|
| 284 |
+
Step 19750/50462 | Loss: 2.5990 | LR: 3.40e-04 | 5218.6ms/step | 11.5 samples/s
|
| 285 |
+
Validation at step 19750: Loss = 2.6302, Perplexity = 13.88
|
| 286 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-19750
|
| 287 |
+
Step 19760/50462 | Loss: 2.6374 | LR: 3.40e-04 | 10133.4ms/step | 5.9 samples/s
|
| 288 |
+
Step 19770/50462 | Loss: 2.6072 | LR: 3.40e-04 | 5201.9ms/step | 11.5 samples/s
|
| 289 |
+
Step 19780/50462 | Loss: 2.6165 | LR: 3.40e-04 | 5192.3ms/step | 11.6 samples/s
|
| 290 |
+
Step 19790/50462 | Loss: 2.6049 | LR: 3.40e-04 | 5201.5ms/step | 11.5 samples/s
|
| 291 |
+
Step 19800/50462 | Loss: 2.6400 | LR: 3.40e-04 | 5208.4ms/step | 11.5 samples/s
|
| 292 |
+
Step 19810/50462 | Loss: 2.6358 | LR: 3.40e-04 | 5208.6ms/step | 11.5 samples/s
|
| 293 |
+
Step 19820/50462 | Loss: 2.6181 | LR: 3.39e-04 | 5211.8ms/step | 11.5 samples/s
|
| 294 |
+
Step 19830/50462 | Loss: 2.6115 | LR: 3.39e-04 | 5210.6ms/step | 11.5 samples/s
|
| 295 |
+
Step 19840/50462 | Loss: 2.6226 | LR: 3.39e-04 | 5208.6ms/step | 11.5 samples/s
|
| 296 |
+
Step 19850/50462 | Loss: 2.6281 | LR: 3.39e-04 | 5216.2ms/step | 11.5 samples/s
|
| 297 |
+
Step 19860/50462 | Loss: 2.5926 | LR: 3.39e-04 | 5211.7ms/step | 11.5 samples/s
|
| 298 |
+
Step 19870/50462 | Loss: 2.6337 | LR: 3.39e-04 | 5213.9ms/step | 11.5 samples/s
|
| 299 |
+
Step 19880/50462 | Loss: 2.6258 | LR: 3.39e-04 | 5203.8ms/step | 11.5 samples/s
|
| 300 |
+
Step 19890/50462 | Loss: 2.6196 | LR: 3.38e-04 | 5206.4ms/step | 11.5 samples/s
|
| 301 |
+
Step 19900/50462 | Loss: 2.6335 | LR: 3.38e-04 | 5211.7ms/step | 11.5 samples/s
|
| 302 |
+
Step 19910/50462 | Loss: 2.6116 | LR: 3.38e-04 | 5212.3ms/step | 11.5 samples/s
|
| 303 |
+
Step 19920/50462 | Loss: 2.6139 | LR: 3.38e-04 | 5217.4ms/step | 11.5 samples/s
|
| 304 |
+
Step 19930/50462 | Loss: 2.6116 | LR: 3.38e-04 | 5213.0ms/step | 11.5 samples/s
|
| 305 |
+
Step 19940/50462 | Loss: 2.6022 | LR: 3.38e-04 | 5214.0ms/step | 11.5 samples/s
|
| 306 |
+
Step 19950/50462 | Loss: 2.6203 | LR: 3.37e-04 | 5216.3ms/step | 11.5 samples/s
|
| 307 |
+
Step 19960/50462 | Loss: 2.6059 | LR: 3.37e-04 | 5213.4ms/step | 11.5 samples/s
|
| 308 |
+
Step 19970/50462 | Loss: 2.6136 | LR: 3.37e-04 | 5215.7ms/step | 11.5 samples/s
|
| 309 |
+
Step 19980/50462 | Loss: 2.6060 | LR: 3.37e-04 | 5211.4ms/step | 11.5 samples/s
|
| 310 |
+
Step 19990/50462 | Loss: 2.6302 | LR: 3.37e-04 | 5219.2ms/step | 11.5 samples/s
|
| 311 |
+
Step 20000/50462 | Loss: 2.6289 | LR: 3.37e-04 | 5215.4ms/step | 11.5 samples/s
|
| 312 |
+
Validation at step 20000: Loss = 2.6269, Perplexity = 13.83
|
| 313 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20000
|
| 314 |
+
Step 20010/50462 | Loss: 2.6132 | LR: 3.37e-04 | 10096.1ms/step | 5.9 samples/s
|
| 315 |
+
Step 20020/50462 | Loss: 2.6169 | LR: 3.36e-04 | 5209.0ms/step | 11.5 samples/s
|
| 316 |
+
Step 20030/50462 | Loss: 2.6383 | LR: 3.36e-04 | 5209.3ms/step | 11.5 samples/s
|
| 317 |
+
Step 20040/50462 | Loss: 2.5934 | LR: 3.36e-04 | 5209.4ms/step | 11.5 samples/s
|
| 318 |
+
Step 20050/50462 | Loss: 2.6217 | LR: 3.36e-04 | 5217.0ms/step | 11.5 samples/s
|
| 319 |
+
Step 20060/50462 | Loss: 2.6177 | LR: 3.36e-04 | 5201.9ms/step | 11.5 samples/s
|
| 320 |
+
Step 20070/50462 | Loss: 2.6223 | LR: 3.36e-04 | 5210.9ms/step | 11.5 samples/s
|
| 321 |
+
Step 20080/50462 | Loss: 2.6101 | LR: 3.36e-04 | 5212.8ms/step | 11.5 samples/s
|
| 322 |
+
Step 20090/50462 | Loss: 2.6152 | LR: 3.35e-04 | 5211.5ms/step | 11.5 samples/s
|
| 323 |
+
Step 20100/50462 | Loss: 2.6061 | LR: 3.35e-04 | 5217.2ms/step | 11.5 samples/s
|
| 324 |
+
Step 20110/50462 | Loss: 2.6024 | LR: 3.35e-04 | 5207.9ms/step | 11.5 samples/s
|
| 325 |
+
Step 20120/50462 | Loss: 2.6510 | LR: 3.35e-04 | 5214.3ms/step | 11.5 samples/s
|
| 326 |
+
Step 20130/50462 | Loss: 2.6158 | LR: 3.35e-04 | 5211.2ms/step | 11.5 samples/s
|
| 327 |
+
Step 20140/50462 | Loss: 2.6115 | LR: 3.35e-04 | 5210.2ms/step | 11.5 samples/s
|
| 328 |
+
Step 20150/50462 | Loss: 2.6288 | LR: 3.35e-04 | 5207.9ms/step | 11.5 samples/s
|
| 329 |
+
Step 20160/50462 | Loss: 2.6027 | LR: 3.34e-04 | 5213.5ms/step | 11.5 samples/s
|
| 330 |
+
Step 20170/50462 | Loss: 2.6014 | LR: 3.34e-04 | 5210.2ms/step | 11.5 samples/s
|
| 331 |
+
Step 20180/50462 | Loss: 2.6079 | LR: 3.34e-04 | 5211.8ms/step | 11.5 samples/s
|
| 332 |
+
Step 20190/50462 | Loss: 2.5964 | LR: 3.34e-04 | 5217.5ms/step | 11.5 samples/s
|
| 333 |
+
Step 20200/50462 | Loss: 2.6267 | LR: 3.34e-04 | 5212.1ms/step | 11.5 samples/s
|
| 334 |
+
Step 20210/50462 | Loss: 2.6348 | LR: 3.34e-04 | 5214.7ms/step | 11.5 samples/s
|
| 335 |
+
Step 20220/50462 | Loss: 2.6126 | LR: 3.33e-04 | 5213.1ms/step | 11.5 samples/s
|
| 336 |
+
Step 20230/50462 | Loss: 2.6279 | LR: 3.33e-04 | 5216.2ms/step | 11.5 samples/s
|
| 337 |
+
Step 20240/50462 | Loss: 2.6147 | LR: 3.33e-04 | 5210.9ms/step | 11.5 samples/s
|
| 338 |
+
Step 20250/50462 | Loss: 2.6301 | LR: 3.33e-04 | 5212.2ms/step | 11.5 samples/s
|
| 339 |
+
Validation at step 20250: Loss = 2.6241, Perplexity = 13.79
|
| 340 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20250
|
| 341 |
+
Step 20260/50462 | Loss: 2.6042 | LR: 3.33e-04 | 10053.0ms/step | 6.0 samples/s
|
| 342 |
+
Step 20270/50462 | Loss: 2.6282 | LR: 3.33e-04 | 5213.1ms/step | 11.5 samples/s
|
| 343 |
+
Step 20280/50462 | Loss: 2.6116 | LR: 3.33e-04 | 5205.2ms/step | 11.5 samples/s
|
| 344 |
+
Step 20290/50462 | Loss: 2.6401 | LR: 3.32e-04 | 5210.7ms/step | 11.5 samples/s
|
| 345 |
+
Step 20300/50462 | Loss: 2.6155 | LR: 3.32e-04 | 5212.4ms/step | 11.5 samples/s
|
| 346 |
+
Step 20310/50462 | Loss: 2.6005 | LR: 3.32e-04 | 5198.0ms/step | 11.5 samples/s
|
| 347 |
+
Step 20320/50462 | Loss: 2.6076 | LR: 3.32e-04 | 5206.6ms/step | 11.5 samples/s
|
| 348 |
+
Step 20330/50462 | Loss: 2.6374 | LR: 3.32e-04 | 5211.5ms/step | 11.5 samples/s
|
| 349 |
+
Step 20340/50462 | Loss: 2.6162 | LR: 3.32e-04 | 5205.3ms/step | 11.5 samples/s
|
| 350 |
+
Step 20350/50462 | Loss: 2.6025 | LR: 3.32e-04 | 5208.6ms/step | 11.5 samples/s
|
| 351 |
+
Step 20360/50462 | Loss: 2.5897 | LR: 3.31e-04 | 5210.7ms/step | 11.5 samples/s
|
| 352 |
+
Step 20370/50462 | Loss: 2.6137 | LR: 3.31e-04 | 5207.8ms/step | 11.5 samples/s
|
| 353 |
+
Step 20380/50462 | Loss: 2.6348 | LR: 3.31e-04 | 5209.4ms/step | 11.5 samples/s
|
| 354 |
+
Step 20390/50462 | Loss: 2.6204 | LR: 3.31e-04 | 5203.2ms/step | 11.5 samples/s
|
| 355 |
+
Step 20400/50462 | Loss: 2.6244 | LR: 3.31e-04 | 5211.3ms/step | 11.5 samples/s
|
| 356 |
+
Step 20410/50462 | Loss: 2.5975 | LR: 3.31e-04 | 5212.8ms/step | 11.5 samples/s
|
| 357 |
+
Step 20420/50462 | Loss: 2.5898 | LR: 3.30e-04 | 5221.2ms/step | 11.5 samples/s
|
| 358 |
+
Step 20430/50462 | Loss: 2.5838 | LR: 3.30e-04 | 5208.3ms/step | 11.5 samples/s
|
| 359 |
+
Step 20440/50462 | Loss: 2.6201 | LR: 3.30e-04 | 5215.3ms/step | 11.5 samples/s
|
| 360 |
+
Step 20450/50462 | Loss: 2.5900 | LR: 3.30e-04 | 5205.1ms/step | 11.5 samples/s
|
| 361 |
+
Step 20460/50462 | Loss: 2.6250 | LR: 3.30e-04 | 5212.1ms/step | 11.5 samples/s
|
| 362 |
+
Step 20470/50462 | Loss: 2.6140 | LR: 3.30e-04 | 5210.3ms/step | 11.5 samples/s
|
| 363 |
+
Step 20480/50462 | Loss: 2.5986 | LR: 3.30e-04 | 5205.8ms/step | 11.5 samples/s
|
| 364 |
+
Step 20490/50462 | Loss: 2.6229 | LR: 3.29e-04 | 5211.0ms/step | 11.5 samples/s
|
| 365 |
+
Step 20500/50462 | Loss: 2.6161 | LR: 3.29e-04 | 5213.9ms/step | 11.5 samples/s
|
| 366 |
+
Validation at step 20500: Loss = 2.6218, Perplexity = 13.76
|
| 367 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500
|
| 368 |
+
Step 20510/50462 | Loss: 2.6112 | LR: 3.29e-04 | 10051.6ms/step | 6.0 samples/s
|
| 369 |
+
Step 20520/50462 | Loss: 2.6233 | LR: 3.29e-04 | 5211.8ms/step | 11.5 samples/s
|
| 370 |
+
Step 20530/50462 | Loss: 2.6291 | LR: 3.29e-04 | 5208.1ms/step | 11.5 samples/s
|
| 371 |
+
Step 20540/50462 | Loss: 2.6162 | LR: 3.29e-04 | 5209.2ms/step | 11.5 samples/s
|
| 372 |
+
Step 20550/50462 | Loss: 2.6237 | LR: 3.29e-04 | 5205.4ms/step | 11.5 samples/s
|
| 373 |
+
Step 20560/50462 | Loss: 2.5945 | LR: 3.28e-04 | 5207.2ms/step | 11.5 samples/s
|
| 374 |
+
Step 20570/50462 | Loss: 2.5894 | LR: 3.28e-04 | 5208.5ms/step | 11.5 samples/s
|
| 375 |
+
Step 20580/50462 | Loss: 2.6280 | LR: 3.28e-04 | 5209.3ms/step | 11.5 samples/s
|
| 376 |
+
Step 20590/50462 | Loss: 2.6126 | LR: 3.28e-04 | 5204.8ms/step | 11.5 samples/s
|
| 377 |
+
Step 20600/50462 | Loss: 2.6064 | LR: 3.28e-04 | 5209.0ms/step | 11.5 samples/s
|
| 378 |
+
Step 20610/50462 | Loss: 2.6273 | LR: 3.28e-04 | 5213.3ms/step | 11.5 samples/s
|
| 379 |
+
Step 20620/50462 | Loss: 2.5983 | LR: 3.27e-04 | 5212.2ms/step | 11.5 samples/s
|
| 380 |
+
Step 20630/50462 | Loss: 2.6366 | LR: 3.27e-04 | 5206.8ms/step | 11.5 samples/s
|
| 381 |
+
Step 20640/50462 | Loss: 2.6148 | LR: 3.27e-04 | 5211.1ms/step | 11.5 samples/s
|
| 382 |
+
Step 20650/50462 | Loss: 2.6098 | LR: 3.27e-04 | 5210.6ms/step | 11.5 samples/s
|
| 383 |
+
Step 20660/50462 | Loss: 2.6029 | LR: 3.27e-04 | 5210.3ms/step | 11.5 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume3.log
ADDED
|
@@ -0,0 +1,686 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0825 13:57:37.862000 140 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0825 13:57:37.862000 140 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0825 13:57:37.862000 140 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0825 13:57:37.862000 140 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 2,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 2gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500
|
| 69 |
+
Checkpoint activations: FalseCheckpoint activations: False
|
| 70 |
+
|
| 71 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
================================================================================
|
| 75 |
+
MODEL ARCHITECTURE
|
| 76 |
+
================================================================================
|
| 77 |
+
Model config:
|
| 78 |
+
vocab_size: 151936
|
| 79 |
+
hidden_size: 1536
|
| 80 |
+
num_hidden_layers: 28
|
| 81 |
+
num_attention_heads: 12
|
| 82 |
+
num_key_value_heads: 4
|
| 83 |
+
head_dim: 128
|
| 84 |
+
intermediate_size: 6144
|
| 85 |
+
max_position_embeddings: 32768
|
| 86 |
+
rope_theta: 1000000
|
| 87 |
+
rope_train_position_scale_set: None
|
| 88 |
+
attention_type: sparse_stochastic
|
| 89 |
+
sparse_block_size: 128
|
| 90 |
+
sparse_num_random_blocks: 31
|
| 91 |
+
sparse_mask_pool_size: 1
|
| 92 |
+
sparse_max_seq_length: 8192
|
| 93 |
+
|
| 94 |
+
Parameter count:
|
| 95 |
+
Total parameters: 1,202,352,640
|
| 96 |
+
Trainable parameters: 1,202,352,640
|
| 97 |
+
Non-trainable parameters: 0
|
| 98 |
+
Model size (BF16): 2.40 GB
|
| 99 |
+
|
| 100 |
+
--------------------------------------------------------------------------------
|
| 101 |
+
LAYER-BY-LAYER SUMMARY
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
================================================================================================================================================================
|
| 104 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 107 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True
|
| 108 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True
|
| 109 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True
|
| 139 |
+
├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True
|
| 140 |
+
================================================================================================================================================================
|
| 141 |
+
Total params: 1,435,726,336
|
| 142 |
+
Trainable params: 1,435,726,336
|
| 143 |
+
Non-trainable params: 0
|
| 144 |
+
Total mult-adds (Units.GIGABYTES): 2.87
|
| 145 |
+
================================================================================================================================================================
|
| 146 |
+
Input size (MB): 0.13
|
| 147 |
+
Forward/backward pass size (MB): 52437.19
|
| 148 |
+
Params size (MB): 2871.45
|
| 149 |
+
Estimated Total Size (MB): 55308.77
|
| 150 |
+
================================================================================================================================================================
|
| 151 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 152 |
+
================================================================================
|
| 153 |
+
|
| 154 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 155 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 156 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 157 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 159 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 160 |
+
Dataset split: 3027764 train, 159356 validation
|
| 161 |
+
Training examples (packed, train split): 3027764
|
| 162 |
+
Validation examples (packed, val split): 159356
|
| 163 |
+
Per-device batch size: 2
|
| 164 |
+
World size (GPUs): 2
|
| 165 |
+
Gradient accumulation steps: 15
|
| 166 |
+
Effective batch size (seqs/step): 60
|
| 167 |
+
Sequence length: 8192
|
| 168 |
+
Tokens per optimizer step: 491,520
|
| 169 |
+
Micro-batches per epoch (per rank): 756941
|
| 170 |
+
Optimizer steps per epoch: 50462
|
| 171 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 172 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20500...
|
| 173 |
+
Preserving checkpoint LR state for exact resume: current=[0.00032927688650345087], schedule_base=[0.0005]
|
| 174 |
+
Resumed from checkpoint at step 20500 (epoch 0, micro_step 281235)
|
| 175 |
+
Resuming training from step 20500
|
| 176 |
+
Resuming wandb run rgmkfpnl
|
| 177 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 178 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 179 |
+
wandb: setting up run rgmkfpnl
|
| 180 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 181 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260825_135931-rgmkfpnl
|
| 182 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 183 |
+
wandb: Resuming run 1.2b dense 8k seed70 2gpu resume
|
| 184 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 185 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 186 |
+
Starting training...
|
| 187 |
+
Validation at step 20500: Loss = 2.6218, Perplexity = 13.76
|
| 188 |
+
wandb: WARNING Tried to log to step 20500 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 189 |
+
Switching attention mode for step 20501: dense (dense step interval)
|
| 190 |
+
Step 20510/50462 | Loss: 2.6112 | LR: 3.29e-04 | 73487.6ms/step | 0.8 samples/s
|
| 191 |
+
wandb: WARNING Tried to log to step 20510 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 192 |
+
Step 20520/50462 | Loss: 2.6233 | LR: 3.29e-04 | 5205.4ms/step | 11.5 samples/s
|
| 193 |
+
wandb: WARNING Tried to log to step 20520 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 194 |
+
Step 20530/50462 | Loss: 2.6291 | LR: 3.29e-04 | 5229.9ms/step | 11.5 samples/s
|
| 195 |
+
wandb: WARNING Tried to log to step 20530 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 196 |
+
Step 20540/50462 | Loss: 2.6162 | LR: 3.29e-04 | 5238.7ms/step | 11.5 samples/s
|
| 197 |
+
wandb: WARNING Tried to log to step 20540 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 198 |
+
Step 20550/50462 | Loss: 2.6236 | LR: 3.29e-04 | 5243.9ms/step | 11.4 samples/s
|
| 199 |
+
wandb: WARNING Tried to log to step 20550 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Step 20560/50462 | Loss: 2.5945 | LR: 3.28e-04 | 5245.3ms/step | 11.4 samples/s
|
| 201 |
+
wandb: WARNING Tried to log to step 20560 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 202 |
+
Step 20570/50462 | Loss: 2.5894 | LR: 3.28e-04 | 5247.0ms/step | 11.4 samples/s
|
| 203 |
+
wandb: WARNING Tried to log to step 20570 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 204 |
+
Step 20580/50462 | Loss: 2.6280 | LR: 3.28e-04 | 5253.8ms/step | 11.4 samples/s
|
| 205 |
+
wandb: WARNING Tried to log to step 20580 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 206 |
+
Step 20590/50462 | Loss: 2.6127 | LR: 3.28e-04 | 5246.6ms/step | 11.4 samples/s
|
| 207 |
+
wandb: WARNING Tried to log to step 20590 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 208 |
+
Step 20600/50462 | Loss: 2.6063 | LR: 3.28e-04 | 5249.4ms/step | 11.4 samples/s
|
| 209 |
+
wandb: WARNING Tried to log to step 20600 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Step 20610/50462 | Loss: 2.6273 | LR: 3.28e-04 | 5231.0ms/step | 11.5 samples/s
|
| 211 |
+
wandb: WARNING Tried to log to step 20610 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 212 |
+
Step 20620/50462 | Loss: 2.5983 | LR: 3.27e-04 | 5223.2ms/step | 11.5 samples/s
|
| 213 |
+
wandb: WARNING Tried to log to step 20620 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Step 20630/50462 | Loss: 2.6366 | LR: 3.27e-04 | 5231.8ms/step | 11.5 samples/s
|
| 215 |
+
wandb: WARNING Tried to log to step 20630 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 216 |
+
Step 20640/50462 | Loss: 2.6148 | LR: 3.27e-04 | 5241.1ms/step | 11.4 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 20640 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 20650/50462 | Loss: 2.6097 | LR: 3.27e-04 | 5239.8ms/step | 11.5 samples/s
|
| 219 |
+
wandb: WARNING Tried to log to step 20650 that is less than the current step 20651. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 220 |
+
Step 20660/50462 | Loss: 2.6029 | LR: 3.27e-04 | 5236.5ms/step | 11.5 samples/s
|
| 221 |
+
Step 20670/50462 | Loss: 2.6037 | LR: 3.27e-04 | 5234.5ms/step | 11.5 samples/s
|
| 222 |
+
Step 20680/50462 | Loss: 2.5932 | LR: 3.27e-04 | 5247.3ms/step | 11.4 samples/s
|
| 223 |
+
Step 20690/50462 | Loss: 2.5839 | LR: 3.26e-04 | 5250.9ms/step | 11.4 samples/s
|
| 224 |
+
Step 20700/50462 | Loss: 2.6363 | LR: 3.26e-04 | 5250.7ms/step | 11.4 samples/s
|
| 225 |
+
Step 20710/50462 | Loss: 2.6164 | LR: 3.26e-04 | 5253.2ms/step | 11.4 samples/s
|
| 226 |
+
Step 20720/50462 | Loss: 2.6060 | LR: 3.26e-04 | 5254.0ms/step | 11.4 samples/s
|
| 227 |
+
Step 20730/50462 | Loss: 2.6053 | LR: 3.26e-04 | 5247.7ms/step | 11.4 samples/s
|
| 228 |
+
Step 20740/50462 | Loss: 2.6442 | LR: 3.26e-04 | 5249.8ms/step | 11.4 samples/s
|
| 229 |
+
Step 20750/50462 | Loss: 2.6224 | LR: 3.26e-04 | 5249.3ms/step | 11.4 samples/s
|
| 230 |
+
Validation at step 20750: Loss = 2.6191, Perplexity = 13.72
|
| 231 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-20750
|
| 232 |
+
Step 20760/50462 | Loss: 2.6160 | LR: 3.25e-04 | 10161.8ms/step | 5.9 samples/s
|
| 233 |
+
Step 20770/50462 | Loss: 2.6229 | LR: 3.25e-04 | 5253.7ms/step | 11.4 samples/s
|
| 234 |
+
Step 20780/50462 | Loss: 2.6009 | LR: 3.25e-04 | 5252.2ms/step | 11.4 samples/s
|
| 235 |
+
Step 20790/50462 | Loss: 2.6164 | LR: 3.25e-04 | 5254.2ms/step | 11.4 samples/s
|
| 236 |
+
Step 20800/50462 | Loss: 2.5992 | LR: 3.25e-04 | 5258.3ms/step | 11.4 samples/s
|
| 237 |
+
Step 20810/50462 | Loss: 2.5972 | LR: 3.25e-04 | 5258.7ms/step | 11.4 samples/s
|
| 238 |
+
Step 20820/50462 | Loss: 2.6085 | LR: 3.24e-04 | 5261.7ms/step | 11.4 samples/s
|
| 239 |
+
Step 20830/50462 | Loss: 2.5909 | LR: 3.24e-04 | 5256.6ms/step | 11.4 samples/s
|
| 240 |
+
Step 20840/50462 | Loss: 2.6176 | LR: 3.24e-04 | 5260.6ms/step | 11.4 samples/s
|
| 241 |
+
Step 20850/50462 | Loss: 2.6036 | LR: 3.24e-04 | 5261.2ms/step | 11.4 samples/s
|
| 242 |
+
Step 20860/50462 | Loss: 2.6108 | LR: 3.24e-04 | 5253.8ms/step | 11.4 samples/s
|
| 243 |
+
Step 20870/50462 | Loss: 2.5858 | LR: 3.24e-04 | 5254.0ms/step | 11.4 samples/s
|
| 244 |
+
Step 20880/50462 | Loss: 2.6262 | LR: 3.24e-04 | 5263.0ms/step | 11.4 samples/s
|
| 245 |
+
Step 20890/50462 | Loss: 2.5930 | LR: 3.23e-04 | 5259.3ms/step | 11.4 samples/s
|
| 246 |
+
Step 20900/50462 | Loss: 2.6070 | LR: 3.23e-04 | 5258.7ms/step | 11.4 samples/s
|
| 247 |
+
Step 20910/50462 | Loss: 2.6087 | LR: 3.23e-04 | 5256.9ms/step | 11.4 samples/s
|
| 248 |
+
Step 20920/50462 | Loss: 2.6149 | LR: 3.23e-04 | 5237.1ms/step | 11.5 samples/s
|
| 249 |
+
Step 20930/50462 | Loss: 2.6183 | LR: 3.23e-04 | 5257.2ms/step | 11.4 samples/s
|
| 250 |
+
Step 20940/50462 | Loss: 2.6188 | LR: 3.23e-04 | 5266.6ms/step | 11.4 samples/s
|
| 251 |
+
Step 20950/50462 | Loss: 2.5984 | LR: 3.23e-04 | 5267.3ms/step | 11.4 samples/s
|
| 252 |
+
Step 20960/50462 | Loss: 2.6118 | LR: 3.22e-04 | 5268.8ms/step | 11.4 samples/s
|
| 253 |
+
Step 20970/50462 | Loss: 2.5773 | LR: 3.22e-04 | 5263.5ms/step | 11.4 samples/s
|
| 254 |
+
Step 20980/50462 | Loss: 2.5792 | LR: 3.22e-04 | 5265.1ms/step | 11.4 samples/s
|
| 255 |
+
Step 20990/50462 | Loss: 2.6189 | LR: 3.22e-04 | 5269.0ms/step | 11.4 samples/s
|
| 256 |
+
Step 21000/50462 | Loss: 2.6017 | LR: 3.22e-04 | 5265.6ms/step | 11.4 samples/s
|
| 257 |
+
Validation at step 21000: Loss = 2.6156, Perplexity = 13.68
|
| 258 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21000
|
| 259 |
+
Step 21010/50462 | Loss: 2.6160 | LR: 3.22e-04 | 10149.1ms/step | 5.9 samples/s
|
| 260 |
+
Step 21020/50462 | Loss: 2.6125 | LR: 3.21e-04 | 5251.7ms/step | 11.4 samples/s
|
| 261 |
+
Step 21030/50462 | Loss: 2.6044 | LR: 3.21e-04 | 5260.8ms/step | 11.4 samples/s
|
| 262 |
+
Step 21040/50462 | Loss: 2.5867 | LR: 3.21e-04 | 5267.4ms/step | 11.4 samples/s
|
| 263 |
+
Step 21050/50462 | Loss: 2.5965 | LR: 3.21e-04 | 5265.4ms/step | 11.4 samples/s
|
| 264 |
+
Step 21060/50462 | Loss: 2.6080 | LR: 3.21e-04 | 5270.1ms/step | 11.4 samples/s
|
| 265 |
+
Step 21070/50462 | Loss: 2.6120 | LR: 3.21e-04 | 5270.5ms/step | 11.4 samples/s
|
| 266 |
+
Step 21080/50462 | Loss: 2.6176 | LR: 3.21e-04 | 5242.5ms/step | 11.4 samples/s
|
| 267 |
+
Step 21090/50462 | Loss: 2.6208 | LR: 3.20e-04 | 5256.4ms/step | 11.4 samples/s
|
| 268 |
+
Step 21100/50462 | Loss: 2.6153 | LR: 3.20e-04 | 5231.1ms/step | 11.5 samples/s
|
| 269 |
+
Step 21110/50462 | Loss: 2.6228 | LR: 3.20e-04 | 5257.0ms/step | 11.4 samples/s
|
| 270 |
+
Step 21120/50462 | Loss: 2.6257 | LR: 3.20e-04 | 5265.2ms/step | 11.4 samples/s
|
| 271 |
+
Step 21130/50462 | Loss: 2.5953 | LR: 3.20e-04 | 5256.5ms/step | 11.4 samples/s
|
| 272 |
+
Step 21140/50462 | Loss: 2.6149 | LR: 3.20e-04 | 5261.0ms/step | 11.4 samples/s
|
| 273 |
+
Step 21150/50462 | Loss: 2.5738 | LR: 3.19e-04 | 5260.1ms/step | 11.4 samples/s
|
| 274 |
+
Step 21160/50462 | Loss: 2.6027 | LR: 3.19e-04 | 5265.8ms/step | 11.4 samples/s
|
| 275 |
+
Step 21170/50462 | Loss: 2.6093 | LR: 3.19e-04 | 5265.0ms/step | 11.4 samples/s
|
| 276 |
+
Step 21180/50462 | Loss: 2.5972 | LR: 3.19e-04 | 5264.6ms/step | 11.4 samples/s
|
| 277 |
+
Step 21190/50462 | Loss: 2.5984 | LR: 3.19e-04 | 5261.6ms/step | 11.4 samples/s
|
| 278 |
+
Step 21200/50462 | Loss: 2.6017 | LR: 3.19e-04 | 5259.2ms/step | 11.4 samples/s
|
| 279 |
+
Step 21210/50462 | Loss: 2.6179 | LR: 3.19e-04 | 5268.5ms/step | 11.4 samples/s
|
| 280 |
+
Step 21220/50462 | Loss: 2.5977 | LR: 3.18e-04 | 5262.3ms/step | 11.4 samples/s
|
| 281 |
+
Step 21230/50462 | Loss: 2.5941 | LR: 3.18e-04 | 5252.0ms/step | 11.4 samples/s
|
| 282 |
+
Step 21240/50462 | Loss: 2.6092 | LR: 3.18e-04 | 5244.4ms/step | 11.4 samples/s
|
| 283 |
+
Step 21250/50462 | Loss: 2.6074 | LR: 3.18e-04 | 5259.3ms/step | 11.4 samples/s
|
| 284 |
+
Validation at step 21250: Loss = 2.6139, Perplexity = 13.65
|
| 285 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21250
|
| 286 |
+
Step 21260/50462 | Loss: 2.6159 | LR: 3.18e-04 | 10146.0ms/step | 5.9 samples/s
|
| 287 |
+
Step 21270/50462 | Loss: 2.6063 | LR: 3.18e-04 | 5234.7ms/step | 11.5 samples/s
|
| 288 |
+
Step 21280/50462 | Loss: 2.6003 | LR: 3.17e-04 | 5245.1ms/step | 11.4 samples/s
|
| 289 |
+
Step 21290/50462 | Loss: 2.5630 | LR: 3.17e-04 | 5259.0ms/step | 11.4 samples/s
|
| 290 |
+
Step 21300/50462 | Loss: 2.6113 | LR: 3.17e-04 | 5255.7ms/step | 11.4 samples/s
|
| 291 |
+
Step 21310/50462 | Loss: 2.6041 | LR: 3.17e-04 | 5258.0ms/step | 11.4 samples/s
|
| 292 |
+
Step 21320/50462 | Loss: 2.6098 | LR: 3.17e-04 | 5254.2ms/step | 11.4 samples/s
|
| 293 |
+
Step 21330/50462 | Loss: 2.5988 | LR: 3.17e-04 | 5255.0ms/step | 11.4 samples/s
|
| 294 |
+
Step 21340/50462 | Loss: 2.5928 | LR: 3.17e-04 | 5259.6ms/step | 11.4 samples/s
|
| 295 |
+
Step 21350/50462 | Loss: 2.6224 | LR: 3.16e-04 | 5268.1ms/step | 11.4 samples/s
|
| 296 |
+
Step 21360/50462 | Loss: 2.6198 | LR: 3.16e-04 | 5260.9ms/step | 11.4 samples/s
|
| 297 |
+
Step 21370/50462 | Loss: 2.6144 | LR: 3.16e-04 | 5254.4ms/step | 11.4 samples/s
|
| 298 |
+
Step 21380/50462 | Loss: 2.6049 | LR: 3.16e-04 | 5247.8ms/step | 11.4 samples/s
|
| 299 |
+
Step 21390/50462 | Loss: 2.6159 | LR: 3.16e-04 | 5218.8ms/step | 11.5 samples/s
|
| 300 |
+
Step 21400/50462 | Loss: 2.6273 | LR: 3.16e-04 | 5217.5ms/step | 11.5 samples/s
|
| 301 |
+
Step 21410/50462 | Loss: 2.5950 | LR: 3.16e-04 | 5215.5ms/step | 11.5 samples/s
|
| 302 |
+
Step 21420/50462 | Loss: 2.6088 | LR: 3.15e-04 | 5210.9ms/step | 11.5 samples/s
|
| 303 |
+
Step 21430/50462 | Loss: 2.5994 | LR: 3.15e-04 | 5213.2ms/step | 11.5 samples/s
|
| 304 |
+
Step 21440/50462 | Loss: 2.5965 | LR: 3.15e-04 | 5213.2ms/step | 11.5 samples/s
|
| 305 |
+
Step 21450/50462 | Loss: 2.5874 | LR: 3.15e-04 | 5242.4ms/step | 11.4 samples/s
|
| 306 |
+
Step 21460/50462 | Loss: 2.6141 | LR: 3.15e-04 | 5248.5ms/step | 11.4 samples/s
|
| 307 |
+
Step 21470/50462 | Loss: 2.6077 | LR: 3.15e-04 | 5252.6ms/step | 11.4 samples/s
|
| 308 |
+
Step 21480/50462 | Loss: 2.5992 | LR: 3.14e-04 | 5259.2ms/step | 11.4 samples/s
|
| 309 |
+
Step 21490/50462 | Loss: 2.6112 | LR: 3.14e-04 | 5260.9ms/step | 11.4 samples/s
|
| 310 |
+
Step 21500/50462 | Loss: 2.6164 | LR: 3.14e-04 | 5266.0ms/step | 11.4 samples/s
|
| 311 |
+
Validation at step 21500: Loss = 2.6104, Perplexity = 13.61
|
| 312 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21500
|
| 313 |
+
Step 21510/50462 | Loss: 2.5989 | LR: 3.14e-04 | 10179.1ms/step | 5.9 samples/s
|
| 314 |
+
Step 21520/50462 | Loss: 2.5899 | LR: 3.14e-04 | 5253.1ms/step | 11.4 samples/s
|
| 315 |
+
Step 21530/50462 | Loss: 2.6009 | LR: 3.14e-04 | 5264.4ms/step | 11.4 samples/s
|
| 316 |
+
Step 21540/50462 | Loss: 2.5802 | LR: 3.14e-04 | 5268.1ms/step | 11.4 samples/s
|
| 317 |
+
Step 21550/50462 | Loss: 2.6108 | LR: 3.13e-04 | 5267.1ms/step | 11.4 samples/s
|
| 318 |
+
Step 21560/50462 | Loss: 2.5729 | LR: 3.13e-04 | 5266.3ms/step | 11.4 samples/s
|
| 319 |
+
Step 21570/50462 | Loss: 2.5971 | LR: 3.13e-04 | 5268.0ms/step | 11.4 samples/s
|
| 320 |
+
Step 21580/50462 | Loss: 2.5911 | LR: 3.13e-04 | 5268.1ms/step | 11.4 samples/s
|
| 321 |
+
Step 21590/50462 | Loss: 2.6012 | LR: 3.13e-04 | 5268.5ms/step | 11.4 samples/s
|
| 322 |
+
Step 21600/50462 | Loss: 2.6141 | LR: 3.13e-04 | 5262.9ms/step | 11.4 samples/s
|
| 323 |
+
Step 21610/50462 | Loss: 2.5978 | LR: 3.12e-04 | 5265.1ms/step | 11.4 samples/s
|
| 324 |
+
Step 21620/50462 | Loss: 2.6168 | LR: 3.12e-04 | 5262.3ms/step | 11.4 samples/s
|
| 325 |
+
Step 21630/50462 | Loss: 2.6069 | LR: 3.12e-04 | 5254.3ms/step | 11.4 samples/s
|
| 326 |
+
Step 21640/50462 | Loss: 2.5944 | LR: 3.12e-04 | 5261.4ms/step | 11.4 samples/s
|
| 327 |
+
Step 21650/50462 | Loss: 2.5747 | LR: 3.12e-04 | 5263.6ms/step | 11.4 samples/s
|
| 328 |
+
Step 21660/50462 | Loss: 2.5932 | LR: 3.12e-04 | 5261.5ms/step | 11.4 samples/s
|
| 329 |
+
Step 21670/50462 | Loss: 2.6050 | LR: 3.12e-04 | 5267.1ms/step | 11.4 samples/s
|
| 330 |
+
Step 21680/50462 | Loss: 2.6057 | LR: 3.11e-04 | 5253.2ms/step | 11.4 samples/s
|
| 331 |
+
Step 21690/50462 | Loss: 2.5947 | LR: 3.11e-04 | 5259.4ms/step | 11.4 samples/s
|
| 332 |
+
Step 21700/50462 | Loss: 2.5844 | LR: 3.11e-04 | 5264.8ms/step | 11.4 samples/s
|
| 333 |
+
Step 21710/50462 | Loss: 2.5913 | LR: 3.11e-04 | 5264.9ms/step | 11.4 samples/s
|
| 334 |
+
Step 21720/50462 | Loss: 2.5991 | LR: 3.11e-04 | 5268.6ms/step | 11.4 samples/s
|
| 335 |
+
Step 21730/50462 | Loss: 2.6112 | LR: 3.11e-04 | 5269.3ms/step | 11.4 samples/s
|
| 336 |
+
Step 21740/50462 | Loss: 2.6083 | LR: 3.10e-04 | 5268.4ms/step | 11.4 samples/s
|
| 337 |
+
Step 21750/50462 | Loss: 2.6004 | LR: 3.10e-04 | 5273.4ms/step | 11.4 samples/s
|
| 338 |
+
Validation at step 21750: Loss = 2.6080, Perplexity = 13.57
|
| 339 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-21750
|
| 340 |
+
Step 21760/50462 | Loss: 2.6074 | LR: 3.10e-04 | 10161.4ms/step | 5.9 samples/s
|
| 341 |
+
Step 21770/50462 | Loss: 2.6016 | LR: 3.10e-04 | 5244.2ms/step | 11.4 samples/s
|
| 342 |
+
Step 21780/50462 | Loss: 2.5773 | LR: 3.10e-04 | 5259.1ms/step | 11.4 samples/s
|
| 343 |
+
Step 21790/50462 | Loss: 2.6089 | LR: 3.10e-04 | 5268.0ms/step | 11.4 samples/s
|
| 344 |
+
Step 21800/50462 | Loss: 2.5855 | LR: 3.10e-04 | 5270.6ms/step | 11.4 samples/s
|
| 345 |
+
Step 21810/50462 | Loss: 2.5769 | LR: 3.09e-04 | 5272.1ms/step | 11.4 samples/s
|
| 346 |
+
Step 21820/50462 | Loss: 2.6056 | LR: 3.09e-04 | 5265.1ms/step | 11.4 samples/s
|
| 347 |
+
Step 21830/50462 | Loss: 2.6061 | LR: 3.09e-04 | 5265.0ms/step | 11.4 samples/s
|
| 348 |
+
Step 21840/50462 | Loss: 2.5733 | LR: 3.09e-04 | 5260.3ms/step | 11.4 samples/s
|
| 349 |
+
Step 21850/50462 | Loss: 2.5809 | LR: 3.09e-04 | 5264.9ms/step | 11.4 samples/s
|
| 350 |
+
Step 21860/50462 | Loss: 2.5939 | LR: 3.09e-04 | 5269.7ms/step | 11.4 samples/s
|
| 351 |
+
Step 21870/50462 | Loss: 2.5784 | LR: 3.08e-04 | 5267.7ms/step | 11.4 samples/s
|
| 352 |
+
Step 21880/50462 | Loss: 2.5816 | LR: 3.08e-04 | 5269.5ms/step | 11.4 samples/s
|
| 353 |
+
Step 21890/50462 | Loss: 2.6046 | LR: 3.08e-04 | 5271.2ms/step | 11.4 samples/s
|
| 354 |
+
Step 21900/50462 | Loss: 2.5729 | LR: 3.08e-04 | 5274.1ms/step | 11.4 samples/s
|
| 355 |
+
Step 21910/50462 | Loss: 2.5749 | LR: 3.08e-04 | 5267.8ms/step | 11.4 samples/s
|
| 356 |
+
Step 21920/50462 | Loss: 2.5677 | LR: 3.08e-04 | 5257.8ms/step | 11.4 samples/s
|
| 357 |
+
Step 21930/50462 | Loss: 2.5807 | LR: 3.08e-04 | 5259.3ms/step | 11.4 samples/s
|
| 358 |
+
Step 21940/50462 | Loss: 2.5948 | LR: 3.07e-04 | 5263.4ms/step | 11.4 samples/s
|
| 359 |
+
Step 21950/50462 | Loss: 2.5698 | LR: 3.07e-04 | 5259.1ms/step | 11.4 samples/s
|
| 360 |
+
Step 21960/50462 | Loss: 2.5905 | LR: 3.07e-04 | 5264.5ms/step | 11.4 samples/s
|
| 361 |
+
Step 21970/50462 | Loss: 2.5827 | LR: 3.07e-04 | 5274.1ms/step | 11.4 samples/s
|
| 362 |
+
Step 21980/50462 | Loss: 2.5938 | LR: 3.07e-04 | 5261.7ms/step | 11.4 samples/s
|
| 363 |
+
Step 21990/50462 | Loss: 2.5951 | LR: 3.07e-04 | 5252.5ms/step | 11.4 samples/s
|
| 364 |
+
Step 22000/50462 | Loss: 2.5888 | LR: 3.06e-04 | 5257.4ms/step | 11.4 samples/s
|
| 365 |
+
Validation at step 22000: Loss = 2.6049, Perplexity = 13.53
|
| 366 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22000
|
| 367 |
+
Step 22010/50462 | Loss: 2.5863 | LR: 3.06e-04 | 10145.8ms/step | 5.9 samples/s
|
| 368 |
+
Step 22020/50462 | Loss: 2.5894 | LR: 3.06e-04 | 5251.8ms/step | 11.4 samples/s
|
| 369 |
+
Step 22030/50462 | Loss: 2.5922 | LR: 3.06e-04 | 5261.6ms/step | 11.4 samples/s
|
| 370 |
+
Step 22040/50462 | Loss: 2.6266 | LR: 3.06e-04 | 5264.4ms/step | 11.4 samples/s
|
| 371 |
+
Step 22050/50462 | Loss: 2.5755 | LR: 3.06e-04 | 5235.4ms/step | 11.5 samples/s
|
| 372 |
+
Step 22060/50462 | Loss: 2.5925 | LR: 3.06e-04 | 5252.3ms/step | 11.4 samples/s
|
| 373 |
+
Step 22070/50462 | Loss: 2.6067 | LR: 3.05e-04 | 5258.2ms/step | 11.4 samples/s
|
| 374 |
+
Step 22080/50462 | Loss: 2.6158 | LR: 3.05e-04 | 5250.7ms/step | 11.4 samples/s
|
| 375 |
+
Step 22090/50462 | Loss: 2.6080 | LR: 3.05e-04 | 5264.0ms/step | 11.4 samples/s
|
| 376 |
+
Step 22100/50462 | Loss: 2.6147 | LR: 3.05e-04 | 5258.5ms/step | 11.4 samples/s
|
| 377 |
+
Step 22110/50462 | Loss: 2.5835 | LR: 3.05e-04 | 5258.6ms/step | 11.4 samples/s
|
| 378 |
+
Step 22120/50462 | Loss: 2.6326 | LR: 3.05e-04 | 5260.8ms/step | 11.4 samples/s
|
| 379 |
+
Step 22130/50462 | Loss: 2.5781 | LR: 3.04e-04 | 5260.7ms/step | 11.4 samples/s
|
| 380 |
+
Step 22140/50462 | Loss: 2.6273 | LR: 3.04e-04 | 5265.4ms/step | 11.4 samples/s
|
| 381 |
+
Step 22150/50462 | Loss: 2.5844 | LR: 3.04e-04 | 5261.1ms/step | 11.4 samples/s
|
| 382 |
+
Step 22160/50462 | Loss: 2.5692 | LR: 3.04e-04 | 5253.6ms/step | 11.4 samples/s
|
| 383 |
+
Step 22170/50462 | Loss: 2.5957 | LR: 3.04e-04 | 5267.5ms/step | 11.4 samples/s
|
| 384 |
+
Step 22180/50462 | Loss: 2.5708 | LR: 3.04e-04 | 5258.5ms/step | 11.4 samples/s
|
| 385 |
+
Step 22190/50462 | Loss: 2.6149 | LR: 3.04e-04 | 5268.5ms/step | 11.4 samples/s
|
| 386 |
+
Step 22200/50462 | Loss: 2.5919 | LR: 3.03e-04 | 5265.0ms/step | 11.4 samples/s
|
| 387 |
+
Step 22210/50462 | Loss: 2.5919 | LR: 3.03e-04 | 5258.6ms/step | 11.4 samples/s
|
| 388 |
+
Step 22220/50462 | Loss: 2.5884 | LR: 3.03e-04 | 5262.2ms/step | 11.4 samples/s
|
| 389 |
+
Step 22230/50462 | Loss: 2.5622 | LR: 3.03e-04 | 5257.7ms/step | 11.4 samples/s
|
| 390 |
+
Step 22240/50462 | Loss: 2.6225 | LR: 3.03e-04 | 5244.9ms/step | 11.4 samples/s
|
| 391 |
+
Step 22250/50462 | Loss: 2.5995 | LR: 3.03e-04 | 5256.9ms/step | 11.4 samples/s
|
| 392 |
+
Validation at step 22250: Loss = 2.6026, Perplexity = 13.50
|
| 393 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22250
|
| 394 |
+
Step 22260/50462 | Loss: 2.5734 | LR: 3.02e-04 | 10160.4ms/step | 5.9 samples/s
|
| 395 |
+
Step 22270/50462 | Loss: 2.5848 | LR: 3.02e-04 | 5263.0ms/step | 11.4 samples/s
|
| 396 |
+
Step 22280/50462 | Loss: 2.5922 | LR: 3.02e-04 | 5266.6ms/step | 11.4 samples/s
|
| 397 |
+
Step 22290/50462 | Loss: 2.5892 | LR: 3.02e-04 | 5259.5ms/step | 11.4 samples/s
|
| 398 |
+
Step 22300/50462 | Loss: 2.5944 | LR: 3.02e-04 | 5256.5ms/step | 11.4 samples/s
|
| 399 |
+
Step 22310/50462 | Loss: 2.5769 | LR: 3.02e-04 | 5264.8ms/step | 11.4 samples/s
|
| 400 |
+
Step 22320/50462 | Loss: 2.6190 | LR: 3.02e-04 | 5261.1ms/step | 11.4 samples/s
|
| 401 |
+
Step 22330/50462 | Loss: 2.5938 | LR: 3.01e-04 | 5263.9ms/step | 11.4 samples/s
|
| 402 |
+
Step 22340/50462 | Loss: 2.6030 | LR: 3.01e-04 | 5268.6ms/step | 11.4 samples/s
|
| 403 |
+
Step 22350/50462 | Loss: 2.5909 | LR: 3.01e-04 | 5255.3ms/step | 11.4 samples/s
|
| 404 |
+
Step 22360/50462 | Loss: 2.5801 | LR: 3.01e-04 | 5248.6ms/step | 11.4 samples/s
|
| 405 |
+
Step 22370/50462 | Loss: 2.5950 | LR: 3.01e-04 | 5269.2ms/step | 11.4 samples/s
|
| 406 |
+
Step 22380/50462 | Loss: 2.5749 | LR: 3.01e-04 | 5263.6ms/step | 11.4 samples/s
|
| 407 |
+
Step 22390/50462 | Loss: 2.5847 | LR: 3.00e-04 | 5258.8ms/step | 11.4 samples/s
|
| 408 |
+
Step 22400/50462 | Loss: 2.5919 | LR: 3.00e-04 | 5264.2ms/step | 11.4 samples/s
|
| 409 |
+
Step 22410/50462 | Loss: 2.6246 | LR: 3.00e-04 | 5259.5ms/step | 11.4 samples/s
|
| 410 |
+
Step 22420/50462 | Loss: 2.5804 | LR: 3.00e-04 | 5271.5ms/step | 11.4 samples/s
|
| 411 |
+
Step 22430/50462 | Loss: 2.5943 | LR: 3.00e-04 | 5272.9ms/step | 11.4 samples/s
|
| 412 |
+
Step 22440/50462 | Loss: 2.5676 | LR: 3.00e-04 | 5266.9ms/step | 11.4 samples/s
|
| 413 |
+
Step 22450/50462 | Loss: 2.6003 | LR: 3.00e-04 | 5265.2ms/step | 11.4 samples/s
|
| 414 |
+
Step 22460/50462 | Loss: 2.5658 | LR: 2.99e-04 | 5267.0ms/step | 11.4 samples/s
|
| 415 |
+
Step 22470/50462 | Loss: 2.5626 | LR: 2.99e-04 | 5259.8ms/step | 11.4 samples/s
|
| 416 |
+
Step 22480/50462 | Loss: 2.5766 | LR: 2.99e-04 | 5251.7ms/step | 11.4 samples/s
|
| 417 |
+
Step 22490/50462 | Loss: 2.5852 | LR: 2.99e-04 | 5259.2ms/step | 11.4 samples/s
|
| 418 |
+
Step 22500/50462 | Loss: 2.5788 | LR: 2.99e-04 | 5263.5ms/step | 11.4 samples/s
|
| 419 |
+
Validation at step 22500: Loss = 2.5996, Perplexity = 13.46
|
| 420 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22500
|
| 421 |
+
Step 22510/50462 | Loss: 2.5989 | LR: 2.99e-04 | 10144.9ms/step | 5.9 samples/s
|
| 422 |
+
Step 22520/50462 | Loss: 2.5642 | LR: 2.98e-04 | 5244.6ms/step | 11.4 samples/s
|
| 423 |
+
Step 22530/50462 | Loss: 2.5969 | LR: 2.98e-04 | 5258.5ms/step | 11.4 samples/s
|
| 424 |
+
Step 22540/50462 | Loss: 2.5846 | LR: 2.98e-04 | 5259.1ms/step | 11.4 samples/s
|
| 425 |
+
Step 22550/50462 | Loss: 2.5898 | LR: 2.98e-04 | 5264.6ms/step | 11.4 samples/s
|
| 426 |
+
Step 22560/50462 | Loss: 2.6055 | LR: 2.98e-04 | 5264.6ms/step | 11.4 samples/s
|
| 427 |
+
Step 22570/50462 | Loss: 2.6023 | LR: 2.98e-04 | 5262.5ms/step | 11.4 samples/s
|
| 428 |
+
Step 22580/50462 | Loss: 2.6035 | LR: 2.98e-04 | 5255.6ms/step | 11.4 samples/s
|
| 429 |
+
Step 22590/50462 | Loss: 2.6083 | LR: 2.97e-04 | 5255.7ms/step | 11.4 samples/s
|
| 430 |
+
Step 22600/50462 | Loss: 2.5752 | LR: 2.97e-04 | 5252.8ms/step | 11.4 samples/s
|
| 431 |
+
Step 22610/50462 | Loss: 2.5696 | LR: 2.97e-04 | 5262.5ms/step | 11.4 samples/s
|
| 432 |
+
Step 22620/50462 | Loss: 2.6038 | LR: 2.97e-04 | 5266.1ms/step | 11.4 samples/s
|
| 433 |
+
Step 22630/50462 | Loss: 2.5854 | LR: 2.97e-04 | 5264.2ms/step | 11.4 samples/s
|
| 434 |
+
Step 22640/50462 | Loss: 2.5761 | LR: 2.97e-04 | 5251.2ms/step | 11.4 samples/s
|
| 435 |
+
Step 22650/50462 | Loss: 2.5773 | LR: 2.96e-04 | 5267.6ms/step | 11.4 samples/s
|
| 436 |
+
Step 22660/50462 | Loss: 2.5884 | LR: 2.96e-04 | 5269.1ms/step | 11.4 samples/s
|
| 437 |
+
Step 22670/50462 | Loss: 2.5719 | LR: 2.96e-04 | 5260.9ms/step | 11.4 samples/s
|
| 438 |
+
Step 22680/50462 | Loss: 2.5738 | LR: 2.96e-04 | 5262.2ms/step | 11.4 samples/s
|
| 439 |
+
Step 22690/50462 | Loss: 2.5776 | LR: 2.96e-04 | 5269.4ms/step | 11.4 samples/s
|
| 440 |
+
Step 22700/50462 | Loss: 2.5859 | LR: 2.96e-04 | 5262.4ms/step | 11.4 samples/s
|
| 441 |
+
Step 22710/50462 | Loss: 2.5766 | LR: 2.95e-04 | 5262.4ms/step | 11.4 samples/s
|
| 442 |
+
Step 22720/50462 | Loss: 2.5836 | LR: 2.95e-04 | 5259.3ms/step | 11.4 samples/s
|
| 443 |
+
Step 22730/50462 | Loss: 2.6038 | LR: 2.95e-04 | 5249.5ms/step | 11.4 samples/s
|
| 444 |
+
Step 22740/50462 | Loss: 2.6143 | LR: 2.95e-04 | 5254.4ms/step | 11.4 samples/s
|
| 445 |
+
Step 22750/50462 | Loss: 2.5793 | LR: 2.95e-04 | 5261.4ms/step | 11.4 samples/s
|
| 446 |
+
Validation at step 22750: Loss = 2.5974, Perplexity = 13.43
|
| 447 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-22750
|
| 448 |
+
Step 22760/50462 | Loss: 2.6008 | LR: 2.95e-04 | 10178.3ms/step | 5.9 samples/s
|
| 449 |
+
Step 22770/50462 | Loss: 2.5736 | LR: 2.95e-04 | 5246.7ms/step | 11.4 samples/s
|
| 450 |
+
Step 22780/50462 | Loss: 2.5999 | LR: 2.94e-04 | 5250.2ms/step | 11.4 samples/s
|
| 451 |
+
Step 22790/50462 | Loss: 2.5771 | LR: 2.94e-04 | 5244.4ms/step | 11.4 samples/s
|
| 452 |
+
Step 22800/50462 | Loss: 2.5982 | LR: 2.94e-04 | 5248.5ms/step | 11.4 samples/s
|
| 453 |
+
Step 22810/50462 | Loss: 2.5987 | LR: 2.94e-04 | 5251.3ms/step | 11.4 samples/s
|
| 454 |
+
Step 22820/50462 | Loss: 2.5649 | LR: 2.94e-04 | 5249.1ms/step | 11.4 samples/s
|
| 455 |
+
Step 22830/50462 | Loss: 2.6157 | LR: 2.94e-04 | 5249.5ms/step | 11.4 samples/s
|
| 456 |
+
Step 22840/50462 | Loss: 2.5676 | LR: 2.93e-04 | 5256.7ms/step | 11.4 samples/s
|
| 457 |
+
Step 22850/50462 | Loss: 2.5994 | LR: 2.93e-04 | 5248.7ms/step | 11.4 samples/s
|
| 458 |
+
Step 22860/50462 | Loss: 2.5954 | LR: 2.93e-04 | 5253.9ms/step | 11.4 samples/s
|
| 459 |
+
Step 22870/50462 | Loss: 2.5975 | LR: 2.93e-04 | 5250.7ms/step | 11.4 samples/s
|
| 460 |
+
Step 22880/50462 | Loss: 2.5859 | LR: 2.93e-04 | 5250.8ms/step | 11.4 samples/s
|
| 461 |
+
Step 22890/50462 | Loss: 2.5689 | LR: 2.93e-04 | 5254.3ms/step | 11.4 samples/s
|
| 462 |
+
Step 22900/50462 | Loss: 2.6003 | LR: 2.93e-04 | 5254.2ms/step | 11.4 samples/s
|
| 463 |
+
Step 22910/50462 | Loss: 2.5851 | LR: 2.92e-04 | 5258.0ms/step | 11.4 samples/s
|
| 464 |
+
Step 22920/50462 | Loss: 2.5803 | LR: 2.92e-04 | 5251.7ms/step | 11.4 samples/s
|
| 465 |
+
Step 22930/50462 | Loss: 2.5798 | LR: 2.92e-04 | 5246.7ms/step | 11.4 samples/s
|
| 466 |
+
Step 22940/50462 | Loss: 2.5810 | LR: 2.92e-04 | 5243.6ms/step | 11.4 samples/s
|
| 467 |
+
Step 22950/50462 | Loss: 2.5955 | LR: 2.92e-04 | 5246.8ms/step | 11.4 samples/s
|
| 468 |
+
Step 22960/50462 | Loss: 2.5917 | LR: 2.92e-04 | 5259.0ms/step | 11.4 samples/s
|
| 469 |
+
Step 22970/50462 | Loss: 2.6244 | LR: 2.91e-04 | 5253.0ms/step | 11.4 samples/s
|
| 470 |
+
Step 22980/50462 | Loss: 2.5927 | LR: 2.91e-04 | 5255.1ms/step | 11.4 samples/s
|
| 471 |
+
Step 22990/50462 | Loss: 2.5821 | LR: 2.91e-04 | 5254.7ms/step | 11.4 samples/s
|
| 472 |
+
Step 23000/50462 | Loss: 2.6071 | LR: 2.91e-04 | 5252.9ms/step | 11.4 samples/s
|
| 473 |
+
Validation at step 23000: Loss = 2.5949, Perplexity = 13.40
|
| 474 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23000
|
| 475 |
+
Step 23010/50462 | Loss: 2.5853 | LR: 2.91e-04 | 10141.2ms/step | 5.9 samples/s
|
| 476 |
+
Step 23020/50462 | Loss: 2.6143 | LR: 2.91e-04 | 5228.8ms/step | 11.5 samples/s
|
| 477 |
+
Step 23030/50462 | Loss: 2.5960 | LR: 2.91e-04 | 5209.4ms/step | 11.5 samples/s
|
| 478 |
+
Step 23040/50462 | Loss: 2.5961 | LR: 2.90e-04 | 5236.3ms/step | 11.5 samples/s
|
| 479 |
+
Step 23050/50462 | Loss: 2.5690 | LR: 2.90e-04 | 5235.8ms/step | 11.5 samples/s
|
| 480 |
+
Step 23060/50462 | Loss: 2.5766 | LR: 2.90e-04 | 5232.8ms/step | 11.5 samples/s
|
| 481 |
+
Step 23070/50462 | Loss: 2.5979 | LR: 2.90e-04 | 5253.4ms/step | 11.4 samples/s
|
| 482 |
+
Step 23080/50462 | Loss: 2.5925 | LR: 2.90e-04 | 5234.8ms/step | 11.5 samples/s
|
| 483 |
+
Step 23090/50462 | Loss: 2.5660 | LR: 2.90e-04 | 5240.0ms/step | 11.5 samples/s
|
| 484 |
+
Step 23100/50462 | Loss: 2.5961 | LR: 2.89e-04 | 5257.7ms/step | 11.4 samples/s
|
| 485 |
+
Step 23110/50462 | Loss: 2.5747 | LR: 2.89e-04 | 5246.4ms/step | 11.4 samples/s
|
| 486 |
+
Step 23120/50462 | Loss: 2.5819 | LR: 2.89e-04 | 5243.8ms/step | 11.4 samples/s
|
| 487 |
+
Step 23130/50462 | Loss: 2.5663 | LR: 2.89e-04 | 5246.9ms/step | 11.4 samples/s
|
| 488 |
+
Step 23140/50462 | Loss: 2.5955 | LR: 2.89e-04 | 5247.0ms/step | 11.4 samples/s
|
| 489 |
+
Step 23150/50462 | Loss: 2.5604 | LR: 2.89e-04 | 5251.1ms/step | 11.4 samples/s
|
| 490 |
+
Step 23160/50462 | Loss: 2.5950 | LR: 2.88e-04 | 5259.8ms/step | 11.4 samples/s
|
| 491 |
+
Step 23170/50462 | Loss: 2.6173 | LR: 2.88e-04 | 5256.4ms/step | 11.4 samples/s
|
| 492 |
+
Step 23180/50462 | Loss: 2.5632 | LR: 2.88e-04 | 5246.2ms/step | 11.4 samples/s
|
| 493 |
+
Step 23190/50462 | Loss: 2.5776 | LR: 2.88e-04 | 5250.4ms/step | 11.4 samples/s
|
| 494 |
+
Step 23200/50462 | Loss: 2.5839 | LR: 2.88e-04 | 5241.4ms/step | 11.4 samples/s
|
| 495 |
+
Step 23210/50462 | Loss: 2.5610 | LR: 2.88e-04 | 5256.8ms/step | 11.4 samples/s
|
| 496 |
+
Step 23220/50462 | Loss: 2.5682 | LR: 2.88e-04 | 5219.3ms/step | 11.5 samples/s
|
| 497 |
+
Step 23230/50462 | Loss: 2.5712 | LR: 2.87e-04 | 5245.6ms/step | 11.4 samples/s
|
| 498 |
+
Step 23240/50462 | Loss: 2.5851 | LR: 2.87e-04 | 5250.9ms/step | 11.4 samples/s
|
| 499 |
+
Step 23250/50462 | Loss: 2.5637 | LR: 2.87e-04 | 5253.8ms/step | 11.4 samples/s
|
| 500 |
+
Validation at step 23250: Loss = 2.5926, Perplexity = 13.37
|
| 501 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23250
|
| 502 |
+
Step 23260/50462 | Loss: 2.5726 | LR: 2.87e-04 | 10160.1ms/step | 5.9 samples/s
|
| 503 |
+
Step 23270/50462 | Loss: 2.5895 | LR: 2.87e-04 | 5244.6ms/step | 11.4 samples/s
|
| 504 |
+
Step 23280/50462 | Loss: 2.5851 | LR: 2.87e-04 | 5255.9ms/step | 11.4 samples/s
|
| 505 |
+
Step 23290/50462 | Loss: 2.5927 | LR: 2.86e-04 | 5264.9ms/step | 11.4 samples/s
|
| 506 |
+
Step 23300/50462 | Loss: 2.5873 | LR: 2.86e-04 | 5260.0ms/step | 11.4 samples/s
|
| 507 |
+
Step 23310/50462 | Loss: 2.5930 | LR: 2.86e-04 | 5267.4ms/step | 11.4 samples/s
|
| 508 |
+
Step 23320/50462 | Loss: 2.5865 | LR: 2.86e-04 | 5250.2ms/step | 11.4 samples/s
|
| 509 |
+
Step 23330/50462 | Loss: 2.5710 | LR: 2.86e-04 | 5259.0ms/step | 11.4 samples/s
|
| 510 |
+
Step 23340/50462 | Loss: 2.5898 | LR: 2.86e-04 | 5255.3ms/step | 11.4 samples/s
|
| 511 |
+
Step 23350/50462 | Loss: 2.5587 | LR: 2.86e-04 | 5257.3ms/step | 11.4 samples/s
|
| 512 |
+
Step 23360/50462 | Loss: 2.5872 | LR: 2.85e-04 | 5260.8ms/step | 11.4 samples/s
|
| 513 |
+
Step 23370/50462 | Loss: 2.5741 | LR: 2.85e-04 | 5256.2ms/step | 11.4 samples/s
|
| 514 |
+
Step 23380/50462 | Loss: 2.5798 | LR: 2.85e-04 | 5261.4ms/step | 11.4 samples/s
|
| 515 |
+
Step 23390/50462 | Loss: 2.5625 | LR: 2.85e-04 | 5261.8ms/step | 11.4 samples/s
|
| 516 |
+
Step 23400/50462 | Loss: 2.5762 | LR: 2.85e-04 | 5262.8ms/step | 11.4 samples/s
|
| 517 |
+
Step 23410/50462 | Loss: 2.5800 | LR: 2.85e-04 | 5261.9ms/step | 11.4 samples/s
|
| 518 |
+
Step 23420/50462 | Loss: 2.5713 | LR: 2.84e-04 | 5257.3ms/step | 11.4 samples/s
|
| 519 |
+
Step 23430/50462 | Loss: 2.5580 | LR: 2.84e-04 | 5262.1ms/step | 11.4 samples/s
|
| 520 |
+
Step 23440/50462 | Loss: 2.5798 | LR: 2.84e-04 | 5265.6ms/step | 11.4 samples/s
|
| 521 |
+
Step 23450/50462 | Loss: 2.5629 | LR: 2.84e-04 | 5261.8ms/step | 11.4 samples/s
|
| 522 |
+
Step 23460/50462 | Loss: 2.5688 | LR: 2.84e-04 | 5258.5ms/step | 11.4 samples/s
|
| 523 |
+
Step 23470/50462 | Loss: 2.5779 | LR: 2.84e-04 | 5257.0ms/step | 11.4 samples/s
|
| 524 |
+
Step 23480/50462 | Loss: 2.6059 | LR: 2.83e-04 | 5259.7ms/step | 11.4 samples/s
|
| 525 |
+
Step 23490/50462 | Loss: 2.6297 | LR: 2.83e-04 | 5259.3ms/step | 11.4 samples/s
|
| 526 |
+
Step 23500/50462 | Loss: 2.5668 | LR: 2.83e-04 | 5256.0ms/step | 11.4 samples/s
|
| 527 |
+
Validation at step 23500: Loss = 2.5905, Perplexity = 13.34
|
| 528 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23500
|
| 529 |
+
Step 23510/50462 | Loss: 2.5825 | LR: 2.83e-04 | 10187.1ms/step | 5.9 samples/s
|
| 530 |
+
Step 23520/50462 | Loss: 2.5825 | LR: 2.83e-04 | 5246.0ms/step | 11.4 samples/s
|
| 531 |
+
Step 23530/50462 | Loss: 2.5526 | LR: 2.83e-04 | 5254.3ms/step | 11.4 samples/s
|
| 532 |
+
Step 23540/50462 | Loss: 2.5740 | LR: 2.83e-04 | 5256.9ms/step | 11.4 samples/s
|
| 533 |
+
Step 23550/50462 | Loss: 2.5652 | LR: 2.82e-04 | 5249.0ms/step | 11.4 samples/s
|
| 534 |
+
Step 23560/50462 | Loss: 2.5853 | LR: 2.82e-04 | 5239.3ms/step | 11.5 samples/s
|
| 535 |
+
Step 23570/50462 | Loss: 2.5812 | LR: 2.82e-04 | 5235.9ms/step | 11.5 samples/s
|
| 536 |
+
Step 23580/50462 | Loss: 2.5890 | LR: 2.82e-04 | 5241.9ms/step | 11.4 samples/s
|
| 537 |
+
Step 23590/50462 | Loss: 2.5674 | LR: 2.82e-04 | 5227.8ms/step | 11.5 samples/s
|
| 538 |
+
Step 23600/50462 | Loss: 2.5863 | LR: 2.82e-04 | 5217.1ms/step | 11.5 samples/s
|
| 539 |
+
Step 23610/50462 | Loss: 2.5677 | LR: 2.81e-04 | 5201.2ms/step | 11.5 samples/s
|
| 540 |
+
Step 23620/50462 | Loss: 2.5653 | LR: 2.81e-04 | 5206.2ms/step | 11.5 samples/s
|
| 541 |
+
Step 23630/50462 | Loss: 2.5869 | LR: 2.81e-04 | 5232.3ms/step | 11.5 samples/s
|
| 542 |
+
Step 23640/50462 | Loss: 2.5539 | LR: 2.81e-04 | 5234.4ms/step | 11.5 samples/s
|
| 543 |
+
Step 23650/50462 | Loss: 2.5493 | LR: 2.81e-04 | 5227.7ms/step | 11.5 samples/s
|
| 544 |
+
Step 23660/50462 | Loss: 2.5773 | LR: 2.81e-04 | 5227.7ms/step | 11.5 samples/s
|
| 545 |
+
Step 23670/50462 | Loss: 2.5690 | LR: 2.81e-04 | 5214.0ms/step | 11.5 samples/s
|
| 546 |
+
Step 23680/50462 | Loss: 2.5809 | LR: 2.80e-04 | 5230.4ms/step | 11.5 samples/s
|
| 547 |
+
Step 23690/50462 | Loss: 2.5686 | LR: 2.80e-04 | 5230.8ms/step | 11.5 samples/s
|
| 548 |
+
Step 23700/50462 | Loss: 2.5698 | LR: 2.80e-04 | 5226.8ms/step | 11.5 samples/s
|
| 549 |
+
Step 23710/50462 | Loss: 2.5800 | LR: 2.80e-04 | 5211.2ms/step | 11.5 samples/s
|
| 550 |
+
Step 23720/50462 | Loss: 2.5895 | LR: 2.80e-04 | 5233.0ms/step | 11.5 samples/s
|
| 551 |
+
Step 23730/50462 | Loss: 2.5735 | LR: 2.80e-04 | 5223.8ms/step | 11.5 samples/s
|
| 552 |
+
Step 23740/50462 | Loss: 2.5926 | LR: 2.79e-04 | 5203.3ms/step | 11.5 samples/s
|
| 553 |
+
Step 23750/50462 | Loss: 2.5848 | LR: 2.79e-04 | 5205.7ms/step | 11.5 samples/s
|
| 554 |
+
Validation at step 23750: Loss = 2.5878, Perplexity = 13.30
|
| 555 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-23750
|
| 556 |
+
Step 23760/50462 | Loss: 2.5821 | LR: 2.79e-04 | 10137.3ms/step | 5.9 samples/s
|
| 557 |
+
Step 23770/50462 | Loss: 2.5880 | LR: 2.79e-04 | 5211.2ms/step | 11.5 samples/s
|
| 558 |
+
Step 23780/50462 | Loss: 2.5421 | LR: 2.79e-04 | 5219.2ms/step | 11.5 samples/s
|
| 559 |
+
Step 23790/50462 | Loss: 2.5625 | LR: 2.79e-04 | 5222.5ms/step | 11.5 samples/s
|
| 560 |
+
Step 23800/50462 | Loss: 2.5682 | LR: 2.78e-04 | 5225.2ms/step | 11.5 samples/s
|
| 561 |
+
Step 23810/50462 | Loss: 2.5905 | LR: 2.78e-04 | 5216.4ms/step | 11.5 samples/s
|
| 562 |
+
Step 23820/50462 | Loss: 2.5850 | LR: 2.78e-04 | 5213.3ms/step | 11.5 samples/s
|
| 563 |
+
Step 23830/50462 | Loss: 2.5642 | LR: 2.78e-04 | 5215.4ms/step | 11.5 samples/s
|
| 564 |
+
Step 23840/50462 | Loss: 2.5845 | LR: 2.78e-04 | 5233.8ms/step | 11.5 samples/s
|
| 565 |
+
Step 23850/50462 | Loss: 2.5567 | LR: 2.78e-04 | 5230.0ms/step | 11.5 samples/s
|
| 566 |
+
Step 23860/50462 | Loss: 2.5698 | LR: 2.78e-04 | 5218.1ms/step | 11.5 samples/s
|
| 567 |
+
Step 23870/50462 | Loss: 2.5532 | LR: 2.77e-04 | 5211.5ms/step | 11.5 samples/s
|
| 568 |
+
Step 23880/50462 | Loss: 2.5835 | LR: 2.77e-04 | 5201.9ms/step | 11.5 samples/s
|
| 569 |
+
Step 23890/50462 | Loss: 2.5565 | LR: 2.77e-04 | 5216.8ms/step | 11.5 samples/s
|
| 570 |
+
Step 23900/50462 | Loss: 2.5736 | LR: 2.77e-04 | 5221.0ms/step | 11.5 samples/s
|
| 571 |
+
Step 23910/50462 | Loss: 2.5594 | LR: 2.77e-04 | 5214.7ms/step | 11.5 samples/s
|
| 572 |
+
Step 23920/50462 | Loss: 2.5899 | LR: 2.77e-04 | 5223.6ms/step | 11.5 samples/s
|
| 573 |
+
Step 23930/50462 | Loss: 2.5588 | LR: 2.76e-04 | 5229.7ms/step | 11.5 samples/s
|
| 574 |
+
Step 23940/50462 | Loss: 2.5760 | LR: 2.76e-04 | 5235.1ms/step | 11.5 samples/s
|
| 575 |
+
Step 23950/50462 | Loss: 2.5849 | LR: 2.76e-04 | 5223.4ms/step | 11.5 samples/s
|
| 576 |
+
Step 23960/50462 | Loss: 2.5932 | LR: 2.76e-04 | 5221.1ms/step | 11.5 samples/s
|
| 577 |
+
Step 23970/50462 | Loss: 2.5874 | LR: 2.76e-04 | 5225.1ms/step | 11.5 samples/s
|
| 578 |
+
Step 23980/50462 | Loss: 2.5517 | LR: 2.76e-04 | 5222.5ms/step | 11.5 samples/s
|
| 579 |
+
Step 23990/50462 | Loss: 2.5754 | LR: 2.75e-04 | 5228.4ms/step | 11.5 samples/s
|
| 580 |
+
Step 24000/50462 | Loss: 2.5795 | LR: 2.75e-04 | 5224.6ms/step | 11.5 samples/s
|
| 581 |
+
Validation at step 24000: Loss = 2.5853, Perplexity = 13.27
|
| 582 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24000
|
| 583 |
+
Step 24010/50462 | Loss: 2.5826 | LR: 2.75e-04 | 10097.1ms/step | 5.9 samples/s
|
| 584 |
+
Step 24020/50462 | Loss: 2.5752 | LR: 2.75e-04 | 5210.7ms/step | 11.5 samples/s
|
| 585 |
+
Step 24030/50462 | Loss: 2.5895 | LR: 2.75e-04 | 5210.5ms/step | 11.5 samples/s
|
| 586 |
+
Step 24040/50462 | Loss: 2.5644 | LR: 2.75e-04 | 5218.1ms/step | 11.5 samples/s
|
| 587 |
+
Step 24050/50462 | Loss: 2.5780 | LR: 2.75e-04 | 5219.7ms/step | 11.5 samples/s
|
| 588 |
+
Step 24060/50462 | Loss: 2.5559 | LR: 2.74e-04 | 5225.6ms/step | 11.5 samples/s
|
| 589 |
+
Step 24070/50462 | Loss: 2.5547 | LR: 2.74e-04 | 5230.0ms/step | 11.5 samples/s
|
| 590 |
+
Step 24080/50462 | Loss: 2.5815 | LR: 2.74e-04 | 5232.6ms/step | 11.5 samples/s
|
| 591 |
+
Step 24090/50462 | Loss: 2.5782 | LR: 2.74e-04 | 5221.1ms/step | 11.5 samples/s
|
| 592 |
+
Step 24100/50462 | Loss: 2.5803 | LR: 2.74e-04 | 5212.7ms/step | 11.5 samples/s
|
| 593 |
+
Step 24110/50462 | Loss: 2.5711 | LR: 2.74e-04 | 5217.0ms/step | 11.5 samples/s
|
| 594 |
+
Step 24120/50462 | Loss: 2.5890 | LR: 2.73e-04 | 5216.1ms/step | 11.5 samples/s
|
| 595 |
+
Step 24130/50462 | Loss: 2.5679 | LR: 2.73e-04 | 5219.3ms/step | 11.5 samples/s
|
| 596 |
+
Step 24140/50462 | Loss: 2.5718 | LR: 2.73e-04 | 5211.2ms/step | 11.5 samples/s
|
| 597 |
+
Step 24150/50462 | Loss: 2.5741 | LR: 2.73e-04 | 5212.5ms/step | 11.5 samples/s
|
| 598 |
+
Step 24160/50462 | Loss: 2.5990 | LR: 2.73e-04 | 5219.9ms/step | 11.5 samples/s
|
| 599 |
+
Step 24170/50462 | Loss: 2.5876 | LR: 2.73e-04 | 5232.2ms/step | 11.5 samples/s
|
| 600 |
+
Step 24180/50462 | Loss: 2.5972 | LR: 2.72e-04 | 5232.9ms/step | 11.5 samples/s
|
| 601 |
+
Step 24190/50462 | Loss: 2.5532 | LR: 2.72e-04 | 5229.8ms/step | 11.5 samples/s
|
| 602 |
+
Step 24200/50462 | Loss: 2.5618 | LR: 2.72e-04 | 5223.7ms/step | 11.5 samples/s
|
| 603 |
+
Step 24210/50462 | Loss: 2.5991 | LR: 2.72e-04 | 5239.3ms/step | 11.5 samples/s
|
| 604 |
+
Step 24220/50462 | Loss: 2.5619 | LR: 2.72e-04 | 5230.2ms/step | 11.5 samples/s
|
| 605 |
+
Step 24230/50462 | Loss: 2.5719 | LR: 2.72e-04 | 5217.9ms/step | 11.5 samples/s
|
| 606 |
+
Step 24240/50462 | Loss: 2.5832 | LR: 2.72e-04 | 5236.8ms/step | 11.5 samples/s
|
| 607 |
+
Step 24250/50462 | Loss: 2.5773 | LR: 2.71e-04 | 5218.7ms/step | 11.5 samples/s
|
| 608 |
+
Validation at step 24250: Loss = 2.5833, Perplexity = 13.24
|
| 609 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24250
|
| 610 |
+
Step 24260/50462 | Loss: 2.5640 | LR: 2.71e-04 | 10099.3ms/step | 5.9 samples/s
|
| 611 |
+
Step 24270/50462 | Loss: 2.5928 | LR: 2.71e-04 | 5225.3ms/step | 11.5 samples/s
|
| 612 |
+
Step 24280/50462 | Loss: 2.5830 | LR: 2.71e-04 | 5233.7ms/step | 11.5 samples/s
|
| 613 |
+
Step 24290/50462 | Loss: 2.5623 | LR: 2.71e-04 | 5236.4ms/step | 11.5 samples/s
|
| 614 |
+
Step 24300/50462 | Loss: 2.5790 | LR: 2.71e-04 | 5217.7ms/step | 11.5 samples/s
|
| 615 |
+
Step 24310/50462 | Loss: 2.5583 | LR: 2.70e-04 | 5234.3ms/step | 11.5 samples/s
|
| 616 |
+
Step 24320/50462 | Loss: 2.5886 | LR: 2.70e-04 | 5238.7ms/step | 11.5 samples/s
|
| 617 |
+
Step 24330/50462 | Loss: 2.5694 | LR: 2.70e-04 | 5232.7ms/step | 11.5 samples/s
|
| 618 |
+
Step 24340/50462 | Loss: 2.5625 | LR: 2.70e-04 | 5219.6ms/step | 11.5 samples/s
|
| 619 |
+
Step 24350/50462 | Loss: 2.5720 | LR: 2.70e-04 | 5233.9ms/step | 11.5 samples/s
|
| 620 |
+
Step 24360/50462 | Loss: 2.5505 | LR: 2.70e-04 | 5239.9ms/step | 11.5 samples/s
|
| 621 |
+
Step 24370/50462 | Loss: 2.5561 | LR: 2.70e-04 | 5240.6ms/step | 11.4 samples/s
|
| 622 |
+
Step 24380/50462 | Loss: 2.5929 | LR: 2.69e-04 | 5238.4ms/step | 11.5 samples/s
|
| 623 |
+
Step 24390/50462 | Loss: 2.5895 | LR: 2.69e-04 | 5233.0ms/step | 11.5 samples/s
|
| 624 |
+
Step 24400/50462 | Loss: 2.5693 | LR: 2.69e-04 | 5243.4ms/step | 11.4 samples/s
|
| 625 |
+
Step 24410/50462 | Loss: 2.5606 | LR: 2.69e-04 | 5239.7ms/step | 11.5 samples/s
|
| 626 |
+
Step 24420/50462 | Loss: 2.5741 | LR: 2.69e-04 | 5212.3ms/step | 11.5 samples/s
|
| 627 |
+
Step 24430/50462 | Loss: 2.5902 | LR: 2.69e-04 | 5227.4ms/step | 11.5 samples/s
|
| 628 |
+
Step 24440/50462 | Loss: 2.5794 | LR: 2.68e-04 | 5214.8ms/step | 11.5 samples/s
|
| 629 |
+
Step 24450/50462 | Loss: 2.5438 | LR: 2.68e-04 | 5217.5ms/step | 11.5 samples/s
|
| 630 |
+
Step 24460/50462 | Loss: 2.5742 | LR: 2.68e-04 | 5240.0ms/step | 11.5 samples/s
|
| 631 |
+
Step 24470/50462 | Loss: 2.5748 | LR: 2.68e-04 | 5227.5ms/step | 11.5 samples/s
|
| 632 |
+
Step 24480/50462 | Loss: 2.5773 | LR: 2.68e-04 | 5233.1ms/step | 11.5 samples/s
|
| 633 |
+
Step 24490/50462 | Loss: 2.5644 | LR: 2.68e-04 | 5239.5ms/step | 11.5 samples/s
|
| 634 |
+
Step 24500/50462 | Loss: 2.5876 | LR: 2.67e-04 | 5228.4ms/step | 11.5 samples/s
|
| 635 |
+
Validation at step 24500: Loss = 2.5811, Perplexity = 13.21
|
| 636 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24500
|
| 637 |
+
Step 24510/50462 | Loss: 2.5880 | LR: 2.67e-04 | 10141.1ms/step | 5.9 samples/s
|
| 638 |
+
Step 24520/50462 | Loss: 2.5981 | LR: 2.67e-04 | 5225.8ms/step | 11.5 samples/s
|
| 639 |
+
Step 24530/50462 | Loss: 2.5603 | LR: 2.67e-04 | 5225.4ms/step | 11.5 samples/s
|
| 640 |
+
Step 24540/50462 | Loss: 2.5961 | LR: 2.67e-04 | 5241.4ms/step | 11.4 samples/s
|
| 641 |
+
Step 24550/50462 | Loss: 2.5534 | LR: 2.67e-04 | 5236.2ms/step | 11.5 samples/s
|
| 642 |
+
Step 24560/50462 | Loss: 2.5905 | LR: 2.67e-04 | 5234.0ms/step | 11.5 samples/s
|
| 643 |
+
Step 24570/50462 | Loss: 2.5791 | LR: 2.66e-04 | 5238.8ms/step | 11.5 samples/s
|
| 644 |
+
Step 24580/50462 | Loss: 2.5699 | LR: 2.66e-04 | 5225.2ms/step | 11.5 samples/s
|
| 645 |
+
Step 24590/50462 | Loss: 2.5891 | LR: 2.66e-04 | 5215.5ms/step | 11.5 samples/s
|
| 646 |
+
Step 24600/50462 | Loss: 2.5722 | LR: 2.66e-04 | 5214.2ms/step | 11.5 samples/s
|
| 647 |
+
Step 24610/50462 | Loss: 2.5772 | LR: 2.66e-04 | 5229.8ms/step | 11.5 samples/s
|
| 648 |
+
Step 24620/50462 | Loss: 2.5845 | LR: 2.66e-04 | 5221.0ms/step | 11.5 samples/s
|
| 649 |
+
Step 24630/50462 | Loss: 2.5606 | LR: 2.65e-04 | 5229.8ms/step | 11.5 samples/s
|
| 650 |
+
Step 24640/50462 | Loss: 2.5854 | LR: 2.65e-04 | 5242.8ms/step | 11.4 samples/s
|
| 651 |
+
Step 24650/50462 | Loss: 2.5641 | LR: 2.65e-04 | 5244.9ms/step | 11.4 samples/s
|
| 652 |
+
Step 24660/50462 | Loss: 2.5776 | LR: 2.65e-04 | 5221.2ms/step | 11.5 samples/s
|
| 653 |
+
Step 24670/50462 | Loss: 2.5805 | LR: 2.65e-04 | 5202.6ms/step | 11.5 samples/s
|
| 654 |
+
Step 24680/50462 | Loss: 2.5672 | LR: 2.65e-04 | 5240.9ms/step | 11.4 samples/s
|
| 655 |
+
Step 24690/50462 | Loss: 2.5786 | LR: 2.64e-04 | 5244.2ms/step | 11.4 samples/s
|
| 656 |
+
Step 24700/50462 | Loss: 2.5703 | LR: 2.64e-04 | 5240.5ms/step | 11.4 samples/s
|
| 657 |
+
Step 24710/50462 | Loss: 2.5919 | LR: 2.64e-04 | 5231.2ms/step | 11.5 samples/s
|
| 658 |
+
Step 24720/50462 | Loss: 2.5526 | LR: 2.64e-04 | 5239.2ms/step | 11.5 samples/s
|
| 659 |
+
Step 24730/50462 | Loss: 2.5615 | LR: 2.64e-04 | 5251.0ms/step | 11.4 samples/s
|
| 660 |
+
Step 24740/50462 | Loss: 2.5601 | LR: 2.64e-04 | 5247.9ms/step | 11.4 samples/s
|
| 661 |
+
Step 24750/50462 | Loss: 2.5584 | LR: 2.64e-04 | 5254.1ms/step | 11.4 samples/s
|
| 662 |
+
Validation at step 24750: Loss = 2.5786, Perplexity = 13.18
|
| 663 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-24750
|
| 664 |
+
Step 24760/50462 | Loss: 2.5698 | LR: 2.63e-04 | 10136.6ms/step | 5.9 samples/s
|
| 665 |
+
Step 24770/50462 | Loss: 2.5647 | LR: 2.63e-04 | 5229.3ms/step | 11.5 samples/s
|
| 666 |
+
Step 24780/50462 | Loss: 2.5581 | LR: 2.63e-04 | 5243.3ms/step | 11.4 samples/s
|
| 667 |
+
Step 24790/50462 | Loss: 2.5728 | LR: 2.63e-04 | 5239.2ms/step | 11.5 samples/s
|
| 668 |
+
Step 24800/50462 | Loss: 2.5682 | LR: 2.63e-04 | 5206.8ms/step | 11.5 samples/s
|
| 669 |
+
Step 24810/50462 | Loss: 2.5696 | LR: 2.63e-04 | 5236.0ms/step | 11.5 samples/s
|
| 670 |
+
Step 24820/50462 | Loss: 2.5859 | LR: 2.62e-04 | 5239.8ms/step | 11.5 samples/s
|
| 671 |
+
Step 24830/50462 | Loss: 2.5520 | LR: 2.62e-04 | 5223.5ms/step | 11.5 samples/s
|
| 672 |
+
Step 24840/50462 | Loss: 2.5834 | LR: 2.62e-04 | 5243.3ms/step | 11.4 samples/s
|
| 673 |
+
Step 24850/50462 | Loss: 2.5677 | LR: 2.62e-04 | 5229.6ms/step | 11.5 samples/s
|
| 674 |
+
Step 24860/50462 | Loss: 2.5612 | LR: 2.62e-04 | 5246.4ms/step | 11.4 samples/s
|
| 675 |
+
Step 24870/50462 | Loss: 2.5683 | LR: 2.62e-04 | 5242.4ms/step | 11.4 samples/s
|
| 676 |
+
Step 24880/50462 | Loss: 2.5503 | LR: 2.61e-04 | 5245.5ms/step | 11.4 samples/s
|
| 677 |
+
Step 24890/50462 | Loss: 2.5759 | LR: 2.61e-04 | 5213.3ms/step | 11.5 samples/s
|
| 678 |
+
Step 24900/50462 | Loss: 2.5666 | LR: 2.61e-04 | 5221.0ms/step | 11.5 samples/s
|
| 679 |
+
Step 24910/50462 | Loss: 2.5689 | LR: 2.61e-04 | 5242.4ms/step | 11.4 samples/s
|
| 680 |
+
Step 24920/50462 | Loss: 2.5463 | LR: 2.61e-04 | 5220.0ms/step | 11.5 samples/s
|
| 681 |
+
Step 24930/50462 | Loss: 2.6253 | LR: 2.61e-04 | 5207.7ms/step | 11.5 samples/s
|
| 682 |
+
Step 24940/50462 | Loss: 2.5709 | LR: 2.61e-04 | 5225.0ms/step | 11.5 samples/s
|
| 683 |
+
Step 24950/50462 | Loss: 2.5711 | LR: 2.60e-04 | 5219.2ms/step | 11.5 samples/s
|
| 684 |
+
Step 24960/50462 | Loss: 2.5634 | LR: 2.60e-04 | 5228.8ms/step | 11.5 samples/s
|
| 685 |
+
Step 24970/50462 | Loss: 2.5673 | LR: 2.60e-04 | 5235.6ms/step | 11.5 samples/s
|
| 686 |
+
Step 24980/50462 | Loss: 2.5675 | LR: 2.60e-04 | 5244.5ms/step | 11.4 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_2gpu_resume_latest.log
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0826 14:02:12.525000 4475 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0826 14:02:12.525000 4475 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0826 14:02:12.525000 4475 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0826 14:02:12.525000 4475 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32000",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 30,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 2gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32000
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
================================================================================
|
| 75 |
+
MODEL ARCHITECTURE
|
| 76 |
+
================================================================================
|
| 77 |
+
Model config:
|
| 78 |
+
vocab_size: 151936
|
| 79 |
+
hidden_size: 1536
|
| 80 |
+
num_hidden_layers: 28
|
| 81 |
+
num_attention_heads: 12
|
| 82 |
+
num_key_value_heads: 4
|
| 83 |
+
head_dim: 128
|
| 84 |
+
intermediate_size: 6144
|
| 85 |
+
max_position_embeddings: 32768
|
| 86 |
+
rope_theta: 1000000
|
| 87 |
+
rope_train_position_scale_set: None
|
| 88 |
+
attention_type: sparse_stochastic
|
| 89 |
+
sparse_block_size: 128
|
| 90 |
+
sparse_num_random_blocks: 31
|
| 91 |
+
sparse_mask_pool_size: 1
|
| 92 |
+
sparse_max_seq_length: 8192
|
| 93 |
+
|
| 94 |
+
Parameter count:
|
| 95 |
+
Total parameters: 1,202,352,640
|
| 96 |
+
Trainable parameters: 1,202,352,640
|
| 97 |
+
Non-trainable parameters: 0
|
| 98 |
+
Model size (BF16): 2.40 GB
|
| 99 |
+
|
| 100 |
+
--------------------------------------------------------------------------------
|
| 101 |
+
LAYER-BY-LAYER SUMMARY
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
================================================================================================================================================================
|
| 104 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 107 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 108 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 109 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 110 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 111 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 139 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 140 |
+
================================================================================================================================================================
|
| 141 |
+
Total params: 1,435,726,336
|
| 142 |
+
Trainable params: 1,435,726,336
|
| 143 |
+
Non-trainable params: 0
|
| 144 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 145 |
+
================================================================================================================================================================
|
| 146 |
+
Input size (MB): 0.07
|
| 147 |
+
Forward/backward pass size (MB): 26218.59
|
| 148 |
+
Params size (MB): 2871.45
|
| 149 |
+
Estimated Total Size (MB): 29090.11
|
| 150 |
+
================================================================================================================================================================
|
| 151 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 152 |
+
================================================================================
|
| 153 |
+
|
| 154 |
+
torch.compile disabled for this run
|
| 155 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 156 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 158 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 160 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 161 |
+
Dataset split: 3027764 train, 159356 validation
|
| 162 |
+
Training examples (packed, train split): 3027764
|
| 163 |
+
Validation examples (packed, val split): 159356
|
| 164 |
+
Per-device batch size: 1
|
| 165 |
+
World size (GPUs): 2
|
| 166 |
+
Gradient accumulation steps: 30
|
| 167 |
+
Effective batch size (seqs/step): 60
|
| 168 |
+
Sequence length: 8192
|
| 169 |
+
Tokens per optimizer step: 491,520
|
| 170 |
+
Micro-batches per epoch (per rank): 1513882
|
| 171 |
+
Optimizer steps per epoch: 50462
|
| 172 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 173 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32000...
|
| 174 |
+
Preserving checkpoint LR state for exact resume: current=[0.0001517080469136949], schedule_base=[0.0005]
|
| 175 |
+
Resumed from checkpoint at step 32000 (epoch 0, micro_step 0)
|
| 176 |
+
Resuming training from step 32000
|
| 177 |
+
Resuming wandb run rgmkfpnl
|
| 178 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 179 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 180 |
+
wandb: setting up run rgmkfpnl
|
| 181 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 182 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_140249-rgmkfpnl
|
| 183 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 184 |
+
wandb: Resuming run 1.2b dense 8k seed70 2gpu resume
|
| 185 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 186 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 187 |
+
Starting training...
|
| 188 |
+
Validation at step 32000: Loss = 2.5320, Perplexity = 12.58
|
| 189 |
+
Switching attention mode for step 32001: dense (dense step interval)
|
| 190 |
+
wandb: WARNING Tried to log to step 32000 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 191 |
+
Step 32010/50462 | Loss: 2.5168 | LR: 1.52e-04 | 14380.0ms/step | 4.2 samples/s
|
| 192 |
+
wandb: WARNING Tried to log to step 32010 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 193 |
+
Step 32020/50462 | Loss: 2.5393 | LR: 1.51e-04 | 7968.8ms/step | 7.5 samples/s
|
| 194 |
+
wandb: WARNING Tried to log to step 32020 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 195 |
+
Step 32030/50462 | Loss: 2.5201 | LR: 1.51e-04 | 7955.3ms/step | 7.5 samples/s
|
| 196 |
+
wandb: WARNING Tried to log to step 32030 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 197 |
+
Step 32040/50462 | Loss: 2.5227 | LR: 1.51e-04 | 7953.5ms/step | 7.5 samples/s
|
| 198 |
+
wandb: WARNING Tried to log to step 32040 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 199 |
+
Step 32050/50462 | Loss: 2.5264 | LR: 1.51e-04 | 7952.9ms/step | 7.5 samples/s
|
| 200 |
+
wandb: WARNING Tried to log to step 32050 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 201 |
+
Step 32060/50462 | Loss: 2.5234 | LR: 1.51e-04 | 7953.5ms/step | 7.5 samples/s
|
| 202 |
+
wandb: WARNING Tried to log to step 32060 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 203 |
+
Step 32070/50462 | Loss: 2.5138 | LR: 1.51e-04 | 7954.4ms/step | 7.5 samples/s
|
| 204 |
+
wandb: WARNING Tried to log to step 32070 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 205 |
+
Step 32080/50462 | Loss: 2.5400 | LR: 1.51e-04 | 7955.2ms/step | 7.5 samples/s
|
| 206 |
+
wandb: WARNING Tried to log to step 32080 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 207 |
+
Step 32090/50462 | Loss: 2.5294 | LR: 1.50e-04 | 7956.8ms/step | 7.5 samples/s
|
| 208 |
+
wandb: WARNING Tried to log to step 32090 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 209 |
+
Step 32100/50462 | Loss: 2.5349 | LR: 1.50e-04 | 7954.5ms/step | 7.5 samples/s
|
| 210 |
+
wandb: WARNING Tried to log to step 32100 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 211 |
+
Step 32110/50462 | Loss: 2.5089 | LR: 1.50e-04 | 7952.9ms/step | 7.5 samples/s
|
| 212 |
+
wandb: WARNING Tried to log to step 32110 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 213 |
+
Step 32120/50462 | Loss: 2.5187 | LR: 1.50e-04 | 7955.4ms/step | 7.5 samples/s
|
| 214 |
+
wandb: WARNING Tried to log to step 32120 that is less than the current step 32121. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Step 32130/50462 | Loss: 2.5054 | LR: 1.50e-04 | 7955.8ms/step | 7.5 samples/s
|
| 216 |
+
Step 32140/50462 | Loss: 2.5293 | LR: 1.50e-04 | 7955.8ms/step | 7.5 samples/s
|
| 217 |
+
Step 32150/50462 | Loss: 2.5470 | LR: 1.50e-04 | 7955.2ms/step | 7.5 samples/s
|
| 218 |
+
Step 32160/50462 | Loss: 2.5272 | LR: 1.49e-04 | 7953.8ms/step | 7.5 samples/s
|
| 219 |
+
Step 32170/50462 | Loss: 2.5062 | LR: 1.49e-04 | 7954.2ms/step | 7.5 samples/s
|
| 220 |
+
Step 32180/50462 | Loss: 2.5364 | LR: 1.49e-04 | 7952.2ms/step | 7.5 samples/s
|
| 221 |
+
Step 32190/50462 | Loss: 2.5128 | LR: 1.49e-04 | 7952.9ms/step | 7.5 samples/s
|
| 222 |
+
Step 32200/50462 | Loss: 2.4884 | LR: 1.49e-04 | 7953.6ms/step | 7.5 samples/s
|
| 223 |
+
Step 32210/50462 | Loss: 2.5220 | LR: 1.49e-04 | 7952.5ms/step | 7.5 samples/s
|
| 224 |
+
Step 32220/50462 | Loss: 2.5136 | LR: 1.49e-04 | 7951.2ms/step | 7.5 samples/s
|
| 225 |
+
Step 32230/50462 | Loss: 2.5225 | LR: 1.48e-04 | 7954.1ms/step | 7.5 samples/s
|
| 226 |
+
Step 32240/50462 | Loss: 2.5340 | LR: 1.48e-04 | 7952.1ms/step | 7.5 samples/s
|
| 227 |
+
Step 32250/50462 | Loss: 2.5188 | LR: 1.48e-04 | 7951.2ms/step | 7.5 samples/s
|
| 228 |
+
Validation at step 32250: Loss = 2.5308, Perplexity = 12.56
|
| 229 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32250
|
| 230 |
+
Step 32260/50462 | Loss: 2.5418 | LR: 1.48e-04 | 15161.9ms/step | 4.0 samples/s
|
| 231 |
+
Step 32270/50462 | Loss: 2.5208 | LR: 1.48e-04 | 7951.6ms/step | 7.5 samples/s
|
| 232 |
+
Step 32280/50462 | Loss: 2.5082 | LR: 1.48e-04 | 7953.3ms/step | 7.5 samples/s
|
| 233 |
+
Step 32290/50462 | Loss: 2.5104 | LR: 1.48e-04 | 7952.8ms/step | 7.5 samples/s
|
| 234 |
+
Step 32300/50462 | Loss: 2.5220 | LR: 1.47e-04 | 7952.8ms/step | 7.5 samples/s
|
| 235 |
+
Step 32310/50462 | Loss: 2.4841 | LR: 1.47e-04 | 7951.5ms/step | 7.5 samples/s
|
| 236 |
+
Step 32320/50462 | Loss: 2.5203 | LR: 1.47e-04 | 7951.6ms/step | 7.5 samples/s
|
| 237 |
+
Step 32330/50462 | Loss: 2.5213 | LR: 1.47e-04 | 7951.5ms/step | 7.5 samples/s
|
| 238 |
+
Step 32340/50462 | Loss: 2.5247 | LR: 1.47e-04 | 7951.4ms/step | 7.5 samples/s
|
| 239 |
+
Step 32350/50462 | Loss: 2.5044 | LR: 1.47e-04 | 7950.4ms/step | 7.5 samples/s
|
| 240 |
+
Step 32360/50462 | Loss: 2.5440 | LR: 1.47e-04 | 7953.0ms/step | 7.5 samples/s
|
| 241 |
+
Step 32370/50462 | Loss: 2.5284 | LR: 1.46e-04 | 7953.7ms/step | 7.5 samples/s
|
| 242 |
+
Step 32380/50462 | Loss: 2.4881 | LR: 1.46e-04 | 7951.9ms/step | 7.5 samples/s
|
| 243 |
+
Step 32390/50462 | Loss: 2.5022 | LR: 1.46e-04 | 7952.0ms/step | 7.5 samples/s
|
| 244 |
+
Step 32400/50462 | Loss: 2.5156 | LR: 1.46e-04 | 7950.8ms/step | 7.5 samples/s
|
| 245 |
+
Step 32410/50462 | Loss: 2.5264 | LR: 1.46e-04 | 7951.6ms/step | 7.5 samples/s
|
| 246 |
+
Step 32420/50462 | Loss: 2.5098 | LR: 1.46e-04 | 7951.2ms/step | 7.5 samples/s
|
| 247 |
+
Step 32430/50462 | Loss: 2.5245 | LR: 1.45e-04 | 7953.5ms/step | 7.5 samples/s
|
| 248 |
+
Step 32440/50462 | Loss: 2.5116 | LR: 1.45e-04 | 7951.5ms/step | 7.5 samples/s
|
| 249 |
+
Step 32450/50462 | Loss: 2.5223 | LR: 1.45e-04 | 7952.6ms/step | 7.5 samples/s
|
| 250 |
+
Step 32460/50462 | Loss: 2.5081 | LR: 1.45e-04 | 7954.2ms/step | 7.5 samples/s
|
| 251 |
+
Step 32470/50462 | Loss: 2.4938 | LR: 1.45e-04 | 7954.3ms/step | 7.5 samples/s
|
| 252 |
+
Step 32480/50462 | Loss: 2.5192 | LR: 1.45e-04 | 7953.9ms/step | 7.5 samples/s
|
| 253 |
+
Step 32490/50462 | Loss: 2.4950 | LR: 1.45e-04 | 7956.5ms/step | 7.5 samples/s
|
| 254 |
+
Step 32500/50462 | Loss: 2.5168 | LR: 1.44e-04 | 7958.4ms/step | 7.5 samples/s
|
| 255 |
+
Validation at step 32500: Loss = 2.5298, Perplexity = 12.55
|
| 256 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32500
|
| 257 |
+
Step 32510/50462 | Loss: 2.5303 | LR: 1.44e-04 | 14978.8ms/step | 4.0 samples/s
|
| 258 |
+
Step 32520/50462 | Loss: 2.5262 | LR: 1.44e-04 | 7951.0ms/step | 7.5 samples/s
|
| 259 |
+
Step 32530/50462 | Loss: 2.5260 | LR: 1.44e-04 | 7957.8ms/step | 7.5 samples/s
|
| 260 |
+
Step 32540/50462 | Loss: 2.5123 | LR: 1.44e-04 | 7962.7ms/step | 7.5 samples/s
|
| 261 |
+
Step 32550/50462 | Loss: 2.5382 | LR: 1.44e-04 | 7963.3ms/step | 7.5 samples/s
|
| 262 |
+
Step 32560/50462 | Loss: 2.5191 | LR: 1.44e-04 | 7963.1ms/step | 7.5 samples/s
|
| 263 |
+
Step 32570/50462 | Loss: 2.5264 | LR: 1.43e-04 | 7965.9ms/step | 7.5 samples/s
|
| 264 |
+
Step 32580/50462 | Loss: 2.5139 | LR: 1.43e-04 | 7974.8ms/step | 7.5 samples/s
|
| 265 |
+
Step 32590/50462 | Loss: 2.5012 | LR: 1.43e-04 | 7958.3ms/step | 7.5 samples/s
|
| 266 |
+
Step 32600/50462 | Loss: 2.5204 | LR: 1.43e-04 | 7957.7ms/step | 7.5 samples/s
|
| 267 |
+
Step 32610/50462 | Loss: 2.4918 | LR: 1.43e-04 | 7957.4ms/step | 7.5 samples/s
|
| 268 |
+
Step 32620/50462 | Loss: 2.5209 | LR: 1.43e-04 | 7957.7ms/step | 7.5 samples/s
|
| 269 |
+
Step 32630/50462 | Loss: 2.5136 | LR: 1.43e-04 | 7957.4ms/step | 7.5 samples/s
|
| 270 |
+
Step 32640/50462 | Loss: 2.4992 | LR: 1.42e-04 | 7958.1ms/step | 7.5 samples/s
|
| 271 |
+
Step 32650/50462 | Loss: 2.5108 | LR: 1.42e-04 | 7959.3ms/step | 7.5 samples/s
|
| 272 |
+
Step 32660/50462 | Loss: 2.4910 | LR: 1.42e-04 | 7954.9ms/step | 7.5 samples/s
|
| 273 |
+
Step 32670/50462 | Loss: 2.5155 | LR: 1.42e-04 | 7957.1ms/step | 7.5 samples/s
|
| 274 |
+
Step 32680/50462 | Loss: 2.5118 | LR: 1.42e-04 | 7956.2ms/step | 7.5 samples/s
|
| 275 |
+
Step 32690/50462 | Loss: 2.5083 | LR: 1.42e-04 | 7956.6ms/step | 7.5 samples/s
|
| 276 |
+
Step 32700/50462 | Loss: 2.5080 | LR: 1.42e-04 | 7955.3ms/step | 7.5 samples/s
|
| 277 |
+
Step 32710/50462 | Loss: 2.5027 | LR: 1.41e-04 | 7953.8ms/step | 7.5 samples/s
|
| 278 |
+
Step 32720/50462 | Loss: 2.5381 | LR: 1.41e-04 | 7953.4ms/step | 7.5 samples/s
|
| 279 |
+
Step 32730/50462 | Loss: 2.5032 | LR: 1.41e-04 | 7952.4ms/step | 7.5 samples/s
|
| 280 |
+
Step 32740/50462 | Loss: 2.5122 | LR: 1.41e-04 | 7952.8ms/step | 7.5 samples/s
|
| 281 |
+
Step 32750/50462 | Loss: 2.5349 | LR: 1.41e-04 | 7969.1ms/step | 7.5 samples/s
|
| 282 |
+
Validation at step 32750: Loss = 2.5289, Perplexity = 12.54
|
| 283 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-32750
|
| 284 |
+
Step 32760/50462 | Loss: 2.4937 | LR: 1.41e-04 | 15038.0ms/step | 4.0 samples/s
|
| 285 |
+
Step 32770/50462 | Loss: 2.5269 | LR: 1.41e-04 | 7973.3ms/step | 7.5 samples/s
|
| 286 |
+
Step 32780/50462 | Loss: 2.5209 | LR: 1.41e-04 | 7961.5ms/step | 7.5 samples/s
|
| 287 |
+
Step 32790/50462 | Loss: 2.5174 | LR: 1.40e-04 | 7968.3ms/step | 7.5 samples/s
|
| 288 |
+
Step 32800/50462 | Loss: 2.5169 | LR: 1.40e-04 | 7966.2ms/step | 7.5 samples/s
|
| 289 |
+
Step 32810/50462 | Loss: 2.4942 | LR: 1.40e-04 | 7958.4ms/step | 7.5 samples/s
|
| 290 |
+
Step 32820/50462 | Loss: 2.5293 | LR: 1.40e-04 | 7973.1ms/step | 7.5 samples/s
|
| 291 |
+
Step 32830/50462 | Loss: 2.5132 | LR: 1.40e-04 | 7966.9ms/step | 7.5 samples/s
|
| 292 |
+
Step 32840/50462 | Loss: 2.5248 | LR: 1.40e-04 | 7962.6ms/step | 7.5 samples/s
|
| 293 |
+
Step 32850/50462 | Loss: 2.5003 | LR: 1.40e-04 | 7970.2ms/step | 7.5 samples/s
|
| 294 |
+
Step 32860/50462 | Loss: 2.5061 | LR: 1.39e-04 | 7973.4ms/step | 7.5 samples/s
|
| 295 |
+
Step 32870/50462 | Loss: 2.4807 | LR: 1.39e-04 | 7965.5ms/step | 7.5 samples/s
|
| 296 |
+
Step 32880/50462 | Loss: 2.4954 | LR: 1.39e-04 | 7967.8ms/step | 7.5 samples/s
|
| 297 |
+
Step 32890/50462 | Loss: 2.5137 | LR: 1.39e-04 | 7968.6ms/step | 7.5 samples/s
|
| 298 |
+
Step 32900/50462 | Loss: 2.5083 | LR: 1.39e-04 | 7962.0ms/step | 7.5 samples/s
|
| 299 |
+
Step 32910/50462 | Loss: 2.5169 | LR: 1.39e-04 | 7967.8ms/step | 7.5 samples/s
|
| 300 |
+
Step 32920/50462 | Loss: 2.5215 | LR: 1.39e-04 | 7966.9ms/step | 7.5 samples/s
|
| 301 |
+
Step 32930/50462 | Loss: 2.5084 | LR: 1.38e-04 | 7965.0ms/step | 7.5 samples/s
|
| 302 |
+
Step 32940/50462 | Loss: 2.5293 | LR: 1.38e-04 | 7968.1ms/step | 7.5 samples/s
|
| 303 |
+
Step 32950/50462 | Loss: 2.5226 | LR: 1.38e-04 | 7966.7ms/step | 7.5 samples/s
|
| 304 |
+
Step 32960/50462 | Loss: 2.5018 | LR: 1.38e-04 | 7972.1ms/step | 7.5 samples/s
|
| 305 |
+
Step 32970/50462 | Loss: 2.5206 | LR: 1.38e-04 | 7979.9ms/step | 7.5 samples/s
|
| 306 |
+
Step 32980/50462 | Loss: 2.5245 | LR: 1.38e-04 | 7985.1ms/step | 7.5 samples/s
|
| 307 |
+
Step 32990/50462 | Loss: 2.5348 | LR: 1.38e-04 | 7982.5ms/step | 7.5 samples/s
|
| 308 |
+
Step 33000/50462 | Loss: 2.5241 | LR: 1.37e-04 | 7975.0ms/step | 7.5 samples/s
|
| 309 |
+
Validation at step 33000: Loss = 2.5279, Perplexity = 12.53
|
| 310 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33000
|
| 311 |
+
Step 33010/50462 | Loss: 2.5174 | LR: 1.37e-04 | 15020.9ms/step | 4.0 samples/s
|
| 312 |
+
Step 33020/50462 | Loss: 2.5204 | LR: 1.37e-04 | 7964.6ms/step | 7.5 samples/s
|
| 313 |
+
Step 33030/50462 | Loss: 2.5059 | LR: 1.37e-04 | 7972.3ms/step | 7.5 samples/s
|
| 314 |
+
Step 33040/50462 | Loss: 2.5184 | LR: 1.37e-04 | 7970.3ms/step | 7.5 samples/s
|
| 315 |
+
Step 33050/50462 | Loss: 2.5247 | LR: 1.37e-04 | 7968.8ms/step | 7.5 samples/s
|
| 316 |
+
Step 33060/50462 | Loss: 2.5153 | LR: 1.37e-04 | 7969.8ms/step | 7.5 samples/s
|
| 317 |
+
Step 33070/50462 | Loss: 2.5172 | LR: 1.36e-04 | 7964.1ms/step | 7.5 samples/s
|
| 318 |
+
Step 33080/50462 | Loss: 2.5123 | LR: 1.36e-04 | 7981.1ms/step | 7.5 samples/s
|
| 319 |
+
Step 33090/50462 | Loss: 2.5081 | LR: 1.36e-04 | 7982.1ms/step | 7.5 samples/s
|
| 320 |
+
Step 33100/50462 | Loss: 2.4931 | LR: 1.36e-04 | 7968.4ms/step | 7.5 samples/s
|
| 321 |
+
Step 33110/50462 | Loss: 2.5296 | LR: 1.36e-04 | 7961.2ms/step | 7.5 samples/s
|
| 322 |
+
Step 33120/50462 | Loss: 2.5104 | LR: 1.36e-04 | 7967.5ms/step | 7.5 samples/s
|
| 323 |
+
Step 33130/50462 | Loss: 2.5040 | LR: 1.36e-04 | 7964.4ms/step | 7.5 samples/s
|
| 324 |
+
Step 33140/50462 | Loss: 2.5142 | LR: 1.35e-04 | 7964.6ms/step | 7.5 samples/s
|
| 325 |
+
Step 33150/50462 | Loss: 2.5310 | LR: 1.35e-04 | 7968.3ms/step | 7.5 samples/s
|
| 326 |
+
Step 33160/50462 | Loss: 2.5055 | LR: 1.35e-04 | 7964.9ms/step | 7.5 samples/s
|
| 327 |
+
Step 33170/50462 | Loss: 2.5262 | LR: 1.35e-04 | 7967.4ms/step | 7.5 samples/s
|
| 328 |
+
Step 33180/50462 | Loss: 2.5182 | LR: 1.35e-04 | 7964.2ms/step | 7.5 samples/s
|
| 329 |
+
Step 33190/50462 | Loss: 2.5355 | LR: 1.35e-04 | 7963.1ms/step | 7.5 samples/s
|
| 330 |
+
Step 33200/50462 | Loss: 2.5024 | LR: 1.35e-04 | 7962.0ms/step | 7.5 samples/s
|
| 331 |
+
Step 33210/50462 | Loss: 2.5094 | LR: 1.34e-04 | 7961.3ms/step | 7.5 samples/s
|
| 332 |
+
Step 33220/50462 | Loss: 2.5521 | LR: 1.34e-04 | 7964.9ms/step | 7.5 samples/s
|
| 333 |
+
Step 33230/50462 | Loss: 2.5323 | LR: 1.34e-04 | 7974.3ms/step | 7.5 samples/s
|
| 334 |
+
Step 33240/50462 | Loss: 2.5066 | LR: 1.34e-04 | 7957.4ms/step | 7.5 samples/s
|
| 335 |
+
Step 33250/50462 | Loss: 2.5200 | LR: 1.34e-04 | 7964.7ms/step | 7.5 samples/s
|
| 336 |
+
Validation at step 33250: Loss = 2.5268, Perplexity = 12.51
|
| 337 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250
|
| 338 |
+
Step 33260/50462 | Loss: 2.5115 | LR: 1.34e-04 | 15129.2ms/step | 4.0 samples/s
|
| 339 |
+
Step 33270/50462 | Loss: 2.4846 | LR: 1.34e-04 | 7963.5ms/step | 7.5 samples/s
|
| 340 |
+
Step 33280/50462 | Loss: 2.5425 | LR: 1.33e-04 | 7960.2ms/step | 7.5 samples/s
|
| 341 |
+
Step 33290/50462 | Loss: 2.5074 | LR: 1.33e-04 | 7957.3ms/step | 7.5 samples/s
|
| 342 |
+
Step 33300/50462 | Loss: 2.5234 | LR: 1.33e-04 | 7960.6ms/step | 7.5 samples/s
|
| 343 |
+
Step 33310/50462 | Loss: 2.5243 | LR: 1.33e-04 | 7964.6ms/step | 7.5 samples/s
|
| 344 |
+
Step 33320/50462 | Loss: 2.5178 | LR: 1.33e-04 | 7959.8ms/step | 7.5 samples/s
|
| 345 |
+
Step 33330/50462 | Loss: 2.4968 | LR: 1.33e-04 | 7966.2ms/step | 7.5 samples/s
|
| 346 |
+
Step 33340/50462 | Loss: 2.4934 | LR: 1.33e-04 | 7958.7ms/step | 7.5 samples/s
|
| 347 |
+
Step 33350/50462 | Loss: 2.5080 | LR: 1.32e-04 | 7964.9ms/step | 7.5 samples/s
|
| 348 |
+
Step 33360/50462 | Loss: 2.5233 | LR: 1.32e-04 | 7956.9ms/step | 7.5 samples/s
|
| 349 |
+
Step 33370/50462 | Loss: 2.5398 | LR: 1.32e-04 | 7958.6ms/step | 7.5 samples/s
|
| 350 |
+
Step 33380/50462 | Loss: 2.5112 | LR: 1.32e-04 | 7959.7ms/step | 7.5 samples/s
|
| 351 |
+
Step 33390/50462 | Loss: 2.5045 | LR: 1.32e-04 | 7959.5ms/step | 7.5 samples/s
|
| 352 |
+
Step 33400/50462 | Loss: 2.5173 | LR: 1.32e-04 | 7960.4ms/step | 7.5 samples/s
|
| 353 |
+
Step 33410/50462 | Loss: 2.5159 | LR: 1.32e-04 | 7959.9ms/step | 7.5 samples/s
|
| 354 |
+
Step 33420/50462 | Loss: 2.5118 | LR: 1.32e-04 | 7953.2ms/step | 7.5 samples/s
|
| 355 |
+
Step 33430/50462 | Loss: 2.5228 | LR: 1.31e-04 | 7953.5ms/step | 7.5 samples/s
|
| 356 |
+
Step 33440/50462 | Loss: 2.5355 | LR: 1.31e-04 | 7960.5ms/step | 7.5 samples/s
|
| 357 |
+
Step 33450/50462 | Loss: 2.4778 | LR: 1.31e-04 | 7965.3ms/step | 7.5 samples/s
|
| 358 |
+
Step 33460/50462 | Loss: 2.5028 | LR: 1.31e-04 | 7954.6ms/step | 7.5 samples/s
|
| 359 |
+
Step 33470/50462 | Loss: 2.5052 | LR: 1.31e-04 | 7961.7ms/step | 7.5 samples/s
|
| 360 |
+
Step 33480/50462 | Loss: 2.5034 | LR: 1.31e-04 | 7959.8ms/step | 7.5 samples/s
|
| 361 |
+
Step 33490/50462 | Loss: 2.5164 | LR: 1.31e-04 | 7957.8ms/step | 7.5 samples/s
|
| 362 |
+
Step 33500/50462 | Loss: 2.5020 | LR: 1.30e-04 | 7963.1ms/step | 7.5 samples/s
|
| 363 |
+
W0826 17:29:31.715000 4475 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
|
| 364 |
+
Traceback (most recent call last):
|
| 365 |
+
File "/mloscratch/homes/navasard/conda/bin/torchrun", line 6, in <module>
|
| 366 |
+
sys.exit(main())
|
| 367 |
+
~~~~^^
|
| 368 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
|
| 369 |
+
return f(*args, **kwargs)
|
| 370 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
|
| 371 |
+
run(args)
|
| 372 |
+
~~~^^^^^^
|
| 373 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
|
| 374 |
+
elastic_launch(
|
| 375 |
+
~~~~~~~~~~~~~~~
|
| 376 |
+
config=config,
|
| 377 |
+
~~~~~~~~~~~~~~
|
| 378 |
+
entrypoint=cmd,
|
| 379 |
+
~~~~~~~~~~~~~~~
|
| 380 |
+
)(*cmd_args)
|
| 381 |
+
~^^^^^^^^^^^
|
| 382 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
|
| 383 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 384 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
|
| 385 |
+
result = agent.run()
|
| 386 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
|
| 387 |
+
result = f(*args, **kwargs)
|
| 388 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
|
| 389 |
+
result = self._invoke_run(role)
|
| 390 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
|
| 391 |
+
time.sleep(monitor_interval)
|
| 392 |
+
~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
| 393 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
|
| 394 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 395 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 4475 got signal: 15
|
epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu.log
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
[rank2]: Traceback (most recent call last):
|
| 6 |
+
[rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in <module>
|
| 7 |
+
[rank2]: main()
|
| 8 |
+
[rank2]: ~~~~^^
|
| 9 |
+
[rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1500, in main
|
| 10 |
+
[rank2]: train(config)
|
| 11 |
+
[rank2]: ~~~~~^^^^^^^^
|
| 12 |
+
[rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 889, in train
|
| 13 |
+
[rank2]: rank, local_rank, world_size = setup_distributed()
|
| 14 |
+
[rank2]: ~~~~~~~~~~~~~~~~~^^
|
| 15 |
+
[rank2]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 479, in setup_distributed
|
| 16 |
+
[rank2]: torch.cuda.set_device(local_rank)
|
| 17 |
+
[rank2]: ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
| 18 |
+
[rank2]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/cuda/__init__.py", line 638, in set_device
|
| 19 |
+
[rank2]: torch._C._cuda_setDevice(device)
|
| 20 |
+
[rank2]: ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^
|
| 21 |
+
[rank2]: torch.AcceleratorError: CUDA error: invalid device ordinal
|
| 22 |
+
[rank2]: GPU device may be out of range, do you have enough GPUs?
|
| 23 |
+
[rank2]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
|
| 24 |
+
[rank2]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
|
| 25 |
+
[rank2]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
|
| 26 |
+
|
| 27 |
+
[rank3]: Traceback (most recent call last):
|
| 28 |
+
[rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1503, in <module>
|
| 29 |
+
[rank3]: main()
|
| 30 |
+
[rank3]: ~~~~^^
|
| 31 |
+
[rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1500, in main
|
| 32 |
+
[rank3]: train(config)
|
| 33 |
+
[rank3]: ~~~~~^^^^^^^^
|
| 34 |
+
[rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 889, in train
|
| 35 |
+
[rank3]: rank, local_rank, world_size = setup_distributed()
|
| 36 |
+
[rank3]: ~~~~~~~~~~~~~~~~~^^
|
| 37 |
+
[rank3]: File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 479, in setup_distributed
|
| 38 |
+
[rank3]: torch.cuda.set_device(local_rank)
|
| 39 |
+
[rank3]: ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
| 40 |
+
[rank3]: File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/cuda/__init__.py", line 638, in set_device
|
| 41 |
+
[rank3]: torch._C._cuda_setDevice(device)
|
| 42 |
+
[rank3]: ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^
|
| 43 |
+
[rank3]: torch.AcceleratorError: CUDA error: invalid device ordinal
|
| 44 |
+
[rank3]: GPU device may be out of range, do you have enough GPUs?
|
| 45 |
+
[rank3]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
|
| 46 |
+
[rank3]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
|
| 47 |
+
[rank3]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
|
| 48 |
+
|
| 49 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260822_210048
|
| 50 |
+
Running config:
|
| 51 |
+
{
|
| 52 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 53 |
+
"model_dtype": "bfloat16",
|
| 54 |
+
"init_from_scratch": true,
|
| 55 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 56 |
+
"val_path": null,
|
| 57 |
+
"max_seq_length": 8192,
|
| 58 |
+
"num_workers": 8,
|
| 59 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260822_210048",
|
| 60 |
+
"resume_from_checkpoint": null,
|
| 61 |
+
"force_new_run": false,
|
| 62 |
+
"num_epochs": 1,
|
| 63 |
+
"max_steps": 50462,
|
| 64 |
+
"per_device_batch_size": 1,
|
| 65 |
+
"gradient_accumulation_steps": 15,
|
| 66 |
+
"learning_rate": 0.0005,
|
| 67 |
+
"weight_decay": 0.01,
|
| 68 |
+
"warmup_steps": 750,
|
| 69 |
+
"lr_scheduler": "cosine",
|
| 70 |
+
"max_grad_norm": 1.0,
|
| 71 |
+
"save_steps": 1000,
|
| 72 |
+
"eval_steps": 250,
|
| 73 |
+
"eval_batches": 50,
|
| 74 |
+
"logging_steps": 10,
|
| 75 |
+
"seed": 70,
|
| 76 |
+
"verbose": true,
|
| 77 |
+
"checkpoint_activations": false,
|
| 78 |
+
"strategy": "ddp",
|
| 79 |
+
"mixed_precision": "bf16",
|
| 80 |
+
"wandb_enabled": true,
|
| 81 |
+
"wandb_project": "stochastic_sparsity",
|
| 82 |
+
"wandb_name": "1.2b dense 8k seed70",
|
| 83 |
+
"wandb_tags": [
|
| 84 |
+
"qwen3",
|
| 85 |
+
"1.2b",
|
| 86 |
+
"8k",
|
| 87 |
+
"p2",
|
| 88 |
+
"dense",
|
| 89 |
+
"seed70"
|
| 90 |
+
],
|
| 91 |
+
"hidden_size": 1536,
|
| 92 |
+
"intermediate_size": 6144,
|
| 93 |
+
"num_attention_heads": 12,
|
| 94 |
+
"num_key_value_heads": 4,
|
| 95 |
+
"num_hidden_layers": 28,
|
| 96 |
+
"head_dim": null,
|
| 97 |
+
"attention_dropout": null,
|
| 98 |
+
"rope_train_position_scale_set": null,
|
| 99 |
+
"attention_type": "sparse_stochastic",
|
| 100 |
+
"sparse_block_size": 128,
|
| 101 |
+
"sparse_num_random_blocks": 31,
|
| 102 |
+
"sparse_mask_pool_size": 1,
|
| 103 |
+
"sparse_val_dense_compare": false,
|
| 104 |
+
"sparse_layer_indices": null,
|
| 105 |
+
"sparse_mask_cache_dir": null,
|
| 106 |
+
"sparse_mask_seed": 0,
|
| 107 |
+
"sparse_mixed_nrb_set": null,
|
| 108 |
+
"dense_mode": true,
|
| 109 |
+
"dense_step_intervals": null
|
| 110 |
+
}
|
| 111 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 112 |
+
[rank2]:[W822 21:00:48.486484629 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
| 113 |
+
W0822 21:00:48.740000 2587 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 4592 closing signal SIGTERM
|
| 114 |
+
W0822 21:00:48.742000 2587 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 4593 closing signal SIGTERM
|
| 115 |
+
E0822 21:00:49.058000 2587 site-packages/torch/distributed/elastic/multiprocessing/api.py:986] failed (exitcode: 1) local_rank: 2 (pid: 4594) of binary: /mloscratch/homes/navasard/conda/bin/python
|
| 116 |
+
Traceback (most recent call last):
|
| 117 |
+
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 118 |
+
File "<frozen runpy>", line 88, in _run_code
|
| 119 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 994, in <module>
|
| 120 |
+
main()
|
| 121 |
+
~~~~^^
|
| 122 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
|
| 123 |
+
return f(*args, **kwargs)
|
| 124 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
|
| 125 |
+
run(args)
|
| 126 |
+
~~~^^^^^^
|
| 127 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
|
| 128 |
+
elastic_launch(
|
| 129 |
+
~~~~~~~~~~~~~~~
|
| 130 |
+
config=config,
|
| 131 |
+
~~~~~~~~~~~~~~
|
| 132 |
+
entrypoint=cmd,
|
| 133 |
+
~~~~~~~~~~~~~~~
|
| 134 |
+
)(*cmd_args)
|
| 135 |
+
~^^^^^^^^^^^
|
| 136 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
|
| 137 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 138 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 317, in launch_agent
|
| 139 |
+
raise ChildFailedError(
|
| 140 |
+
...<2 lines>...
|
| 141 |
+
)
|
| 142 |
+
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
|
| 143 |
+
============================================================
|
| 144 |
+
train.py FAILED
|
| 145 |
+
------------------------------------------------------------
|
| 146 |
+
Failures:
|
| 147 |
+
[1]:
|
| 148 |
+
time : 2026-08-22_21:00:48
|
| 149 |
+
host : h100-workspace-2-gpu-0-9
|
| 150 |
+
rank : 3 (local_rank: 3)
|
| 151 |
+
exitcode : 1 (pid: 4595)
|
| 152 |
+
error_file: <N/A>
|
| 153 |
+
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
|
| 154 |
+
[2]:
|
| 155 |
+
time : 2026-08-22_21:00:49
|
| 156 |
+
host : h100-workspace-2-gpu-0-9
|
| 157 |
+
rank : 0 (local_rank: 0)
|
| 158 |
+
exitcode : -15 (pid: 4592)
|
| 159 |
+
error_file: <N/A>
|
| 160 |
+
traceback : Signal 15 (SIGTERM) received by PID 4592
|
| 161 |
+
[3]:
|
| 162 |
+
time : 2026-08-22_21:00:49
|
| 163 |
+
host : h100-workspace-2-gpu-0-9
|
| 164 |
+
rank : 1 (local_rank: 1)
|
| 165 |
+
exitcode : -15 (pid: 4593)
|
| 166 |
+
error_file: <N/A>
|
| 167 |
+
traceback : Signal 15 (SIGTERM) received by PID 4593
|
| 168 |
+
------------------------------------------------------------
|
| 169 |
+
Root Cause (first observed failure):
|
| 170 |
+
[0]:
|
| 171 |
+
time : 2026-08-22_21:00:48
|
| 172 |
+
host : h100-workspace-2-gpu-0-9
|
| 173 |
+
rank : 2 (local_rank: 2)
|
| 174 |
+
exitcode : 1 (pid: 4594)
|
| 175 |
+
error_file: <N/A>
|
| 176 |
+
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
|
| 177 |
+
============================================================
|
epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu_a100_after_eval.log
ADDED
|
@@ -0,0 +1,749 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": null,
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Initializing model from scratch (random weights)...
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
================================================================================
|
| 77 |
+
MODEL ARCHITECTURE
|
| 78 |
+
================================================================================
|
| 79 |
+
Model config:
|
| 80 |
+
vocab_size: 151936
|
| 81 |
+
hidden_size: 1536
|
| 82 |
+
num_hidden_layers: 28
|
| 83 |
+
num_attention_heads: 12
|
| 84 |
+
num_key_value_heads: 4
|
| 85 |
+
head_dim: 128
|
| 86 |
+
intermediate_size: 6144
|
| 87 |
+
max_position_embeddings: 32768
|
| 88 |
+
rope_theta: 1000000
|
| 89 |
+
rope_train_position_scale_set: None
|
| 90 |
+
attention_type: sparse_stochastic
|
| 91 |
+
sparse_block_size: 128
|
| 92 |
+
sparse_num_random_blocks: 31
|
| 93 |
+
sparse_mask_pool_size: 1
|
| 94 |
+
sparse_max_seq_length: 8192
|
| 95 |
+
|
| 96 |
+
Parameter count:
|
| 97 |
+
Total parameters: 1,202,352,640
|
| 98 |
+
Trainable parameters: 1,202,352,640
|
| 99 |
+
Non-trainable parameters: 0
|
| 100 |
+
Model size (BF16): 2.40 GB
|
| 101 |
+
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
LAYER-BY-LAYER SUMMARY
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 109 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 110 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 111 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 141 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 142 |
+
================================================================================================================================================================
|
| 143 |
+
Total params: 1,435,726,336
|
| 144 |
+
Trainable params: 1,435,726,336
|
| 145 |
+
Non-trainable params: 0
|
| 146 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 147 |
+
================================================================================================================================================================
|
| 148 |
+
Input size (MB): 0.07
|
| 149 |
+
Forward/backward pass size (MB): 26218.59
|
| 150 |
+
Params size (MB): 2871.45
|
| 151 |
+
Estimated Total Size (MB): 29090.11
|
| 152 |
+
================================================================================================================================================================
|
| 153 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 154 |
+
================================================================================
|
| 155 |
+
|
| 156 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 163 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 164 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 165 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 166 |
+
Dataset split: 3027764 train, 159356 validation
|
| 167 |
+
Training examples (packed, train split): 3027764
|
| 168 |
+
Validation examples (packed, val split): 159356
|
| 169 |
+
Per-device batch size: 1
|
| 170 |
+
World size (GPUs): 4
|
| 171 |
+
Gradient accumulation steps: 15
|
| 172 |
+
Effective batch size (seqs/step): 60
|
| 173 |
+
Sequence length: 8192
|
| 174 |
+
Tokens per optimizer step: 491,520
|
| 175 |
+
Micro-batches per epoch (per rank): 756941
|
| 176 |
+
Optimizer steps per epoch: 50462
|
| 177 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 178 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 179 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 180 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 181 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_043841-rgmkfpnl
|
| 182 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 183 |
+
wandb: Syncing run 1.2b dense 8k seed70
|
| 184 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 185 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 186 |
+
Starting training...
|
| 187 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 188 |
+
Step 10/50462 | Loss: 12.2299 | LR: 6.67e-06 | 13397.9ms/step | 4.5 samples/s
|
| 189 |
+
Step 20/50462 | Loss: 11.9156 | LR: 1.33e-05 | 6965.1ms/step | 8.6 samples/s
|
| 190 |
+
Step 30/50462 | Loss: 11.0080 | LR: 2.00e-05 | 6966.3ms/step | 8.6 samples/s
|
| 191 |
+
Step 40/50462 | Loss: 10.1412 | LR: 2.67e-05 | 6953.0ms/step | 8.6 samples/s
|
| 192 |
+
Step 50/50462 | Loss: 9.7365 | LR: 3.33e-05 | 6952.0ms/step | 8.6 samples/s
|
| 193 |
+
Step 60/50462 | Loss: 9.2596 | LR: 4.00e-05 | 6956.4ms/step | 8.6 samples/s
|
| 194 |
+
Step 70/50462 | Loss: 8.7069 | LR: 4.67e-05 | 6960.8ms/step | 8.6 samples/s
|
| 195 |
+
Step 80/50462 | Loss: 8.2501 | LR: 5.33e-05 | 6957.5ms/step | 8.6 samples/s
|
| 196 |
+
Step 90/50462 | Loss: 7.8702 | LR: 6.00e-05 | 6961.8ms/step | 8.6 samples/s
|
| 197 |
+
Step 100/50462 | Loss: 7.4724 | LR: 6.67e-05 | 6967.9ms/step | 8.6 samples/s
|
| 198 |
+
Step 110/50462 | Loss: 7.2534 | LR: 7.33e-05 | 6966.4ms/step | 8.6 samples/s
|
| 199 |
+
Step 120/50462 | Loss: 7.0820 | LR: 8.00e-05 | 6971.5ms/step | 8.6 samples/s
|
| 200 |
+
Step 130/50462 | Loss: 6.9967 | LR: 8.67e-05 | 6974.3ms/step | 8.6 samples/s
|
| 201 |
+
Step 140/50462 | Loss: 6.8992 | LR: 9.33e-05 | 6976.3ms/step | 8.6 samples/s
|
| 202 |
+
Step 150/50462 | Loss: 6.7704 | LR: 1.00e-04 | 6973.5ms/step | 8.6 samples/s
|
| 203 |
+
Step 160/50462 | Loss: 6.6579 | LR: 1.07e-04 | 6975.6ms/step | 8.6 samples/s
|
| 204 |
+
Step 170/50462 | Loss: 6.5643 | LR: 1.13e-04 | 6977.5ms/step | 8.6 samples/s
|
| 205 |
+
Step 180/50462 | Loss: 6.4492 | LR: 1.20e-04 | 6976.6ms/step | 8.6 samples/s
|
| 206 |
+
Step 190/50462 | Loss: 6.3763 | LR: 1.27e-04 | 6977.4ms/step | 8.6 samples/s
|
| 207 |
+
Step 200/50462 | Loss: 6.3030 | LR: 1.33e-04 | 6977.5ms/step | 8.6 samples/s
|
| 208 |
+
Step 210/50462 | Loss: 6.2139 | LR: 1.40e-04 | 6977.2ms/step | 8.6 samples/s
|
| 209 |
+
Step 220/50462 | Loss: 6.2008 | LR: 1.47e-04 | 6976.3ms/step | 8.6 samples/s
|
| 210 |
+
Step 230/50462 | Loss: 6.1205 | LR: 1.53e-04 | 6977.5ms/step | 8.6 samples/s
|
| 211 |
+
Step 240/50462 | Loss: 6.0458 | LR: 1.60e-04 | 6976.5ms/step | 8.6 samples/s
|
| 212 |
+
Step 250/50462 | Loss: 6.0234 | LR: 1.67e-04 | 6978.1ms/step | 8.6 samples/s
|
| 213 |
+
Validation at step 250: Loss = 5.9605, Perplexity = 387.82
|
| 214 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-250
|
| 215 |
+
Step 260/50462 | Loss: 5.9211 | LR: 1.73e-04 | 14290.4ms/step | 4.2 samples/s
|
| 216 |
+
Step 270/50462 | Loss: 5.8880 | LR: 1.80e-04 | 6979.7ms/step | 8.6 samples/s
|
| 217 |
+
Step 280/50462 | Loss: 5.8289 | LR: 1.87e-04 | 6978.4ms/step | 8.6 samples/s
|
| 218 |
+
Step 290/50462 | Loss: 5.7598 | LR: 1.93e-04 | 6983.4ms/step | 8.6 samples/s
|
| 219 |
+
Step 300/50462 | Loss: 5.7123 | LR: 2.00e-04 | 6978.2ms/step | 8.6 samples/s
|
| 220 |
+
Step 310/50462 | Loss: 5.6371 | LR: 2.07e-04 | 6974.8ms/step | 8.6 samples/s
|
| 221 |
+
Step 320/50462 | Loss: 5.6010 | LR: 2.13e-04 | 6974.8ms/step | 8.6 samples/s
|
| 222 |
+
Step 330/50462 | Loss: 5.5710 | LR: 2.20e-04 | 6978.6ms/step | 8.6 samples/s
|
| 223 |
+
Step 340/50462 | Loss: 5.5359 | LR: 2.27e-04 | 6974.4ms/step | 8.6 samples/s
|
| 224 |
+
Step 350/50462 | Loss: 5.4892 | LR: 2.33e-04 | 6973.2ms/step | 8.6 samples/s
|
| 225 |
+
Step 360/50462 | Loss: 5.4533 | LR: 2.40e-04 | 6972.2ms/step | 8.6 samples/s
|
| 226 |
+
Step 370/50462 | Loss: 5.3670 | LR: 2.47e-04 | 6974.5ms/step | 8.6 samples/s
|
| 227 |
+
Step 380/50462 | Loss: 5.3213 | LR: 2.53e-04 | 6973.7ms/step | 8.6 samples/s
|
| 228 |
+
Step 390/50462 | Loss: 5.2728 | LR: 2.60e-04 | 6976.0ms/step | 8.6 samples/s
|
| 229 |
+
Step 400/50462 | Loss: 5.2346 | LR: 2.67e-04 | 6992.1ms/step | 8.6 samples/s
|
| 230 |
+
Step 410/50462 | Loss: 5.2008 | LR: 2.73e-04 | 7002.5ms/step | 8.6 samples/s
|
| 231 |
+
Step 420/50462 | Loss: 5.1574 | LR: 2.80e-04 | 7002.8ms/step | 8.6 samples/s
|
| 232 |
+
Step 430/50462 | Loss: 5.1082 | LR: 2.87e-04 | 7004.1ms/step | 8.6 samples/s
|
| 233 |
+
Step 440/50462 | Loss: 5.0639 | LR: 2.93e-04 | 7001.7ms/step | 8.6 samples/s
|
| 234 |
+
Step 450/50462 | Loss: 5.0605 | LR: 3.00e-04 | 6992.6ms/step | 8.6 samples/s
|
| 235 |
+
Step 460/50462 | Loss: 5.0267 | LR: 3.07e-04 | 7001.6ms/step | 8.6 samples/s
|
| 236 |
+
Step 470/50462 | Loss: 4.9638 | LR: 3.13e-04 | 6999.0ms/step | 8.6 samples/s
|
| 237 |
+
Step 480/50462 | Loss: 4.9458 | LR: 3.20e-04 | 6985.4ms/step | 8.6 samples/s
|
| 238 |
+
Step 490/50462 | Loss: 4.8491 | LR: 3.27e-04 | 6980.0ms/step | 8.6 samples/s
|
| 239 |
+
Step 500/50462 | Loss: 4.8597 | LR: 3.33e-04 | 6981.3ms/step | 8.6 samples/s
|
| 240 |
+
Validation at step 500: Loss = 4.8557, Perplexity = 128.48
|
| 241 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-500
|
| 242 |
+
Step 510/50462 | Loss: 4.8445 | LR: 3.40e-04 | 10743.5ms/step | 5.6 samples/s
|
| 243 |
+
Step 520/50462 | Loss: 4.7613 | LR: 3.47e-04 | 6994.9ms/step | 8.6 samples/s
|
| 244 |
+
Step 530/50462 | Loss: 4.7293 | LR: 3.53e-04 | 6997.9ms/step | 8.6 samples/s
|
| 245 |
+
Step 540/50462 | Loss: 4.7942 | LR: 3.60e-04 | 6991.0ms/step | 8.6 samples/s
|
| 246 |
+
Step 550/50462 | Loss: 4.7975 | LR: 3.67e-04 | 6995.8ms/step | 8.6 samples/s
|
| 247 |
+
Step 560/50462 | Loss: 4.7114 | LR: 3.73e-04 | 6991.5ms/step | 8.6 samples/s
|
| 248 |
+
Step 570/50462 | Loss: 4.6380 | LR: 3.80e-04 | 6992.5ms/step | 8.6 samples/s
|
| 249 |
+
Step 580/50462 | Loss: 4.6088 | LR: 3.87e-04 | 6995.6ms/step | 8.6 samples/s
|
| 250 |
+
Step 590/50462 | Loss: 4.6519 | LR: 3.93e-04 | 6993.0ms/step | 8.6 samples/s
|
| 251 |
+
Step 600/50462 | Loss: 4.5644 | LR: 4.00e-04 | 6992.5ms/step | 8.6 samples/s
|
| 252 |
+
Step 610/50462 | Loss: 4.5056 | LR: 4.07e-04 | 6989.8ms/step | 8.6 samples/s
|
| 253 |
+
Step 620/50462 | Loss: 4.5163 | LR: 4.13e-04 | 6991.3ms/step | 8.6 samples/s
|
| 254 |
+
Step 630/50462 | Loss: 4.5185 | LR: 4.20e-04 | 6993.7ms/step | 8.6 samples/s
|
| 255 |
+
Step 640/50462 | Loss: 4.4487 | LR: 4.27e-04 | 6991.2ms/step | 8.6 samples/s
|
| 256 |
+
Step 650/50462 | Loss: 4.3603 | LR: 4.33e-04 | 6993.4ms/step | 8.6 samples/s
|
| 257 |
+
Step 660/50462 | Loss: 4.3409 | LR: 4.40e-04 | 6994.4ms/step | 8.6 samples/s
|
| 258 |
+
Step 670/50462 | Loss: 4.4596 | LR: 4.47e-04 | 6992.8ms/step | 8.6 samples/s
|
| 259 |
+
Step 680/50462 | Loss: 4.3716 | LR: 4.53e-04 | 6994.4ms/step | 8.6 samples/s
|
| 260 |
+
Step 690/50462 | Loss: 4.2619 | LR: 4.60e-04 | 6997.6ms/step | 8.6 samples/s
|
| 261 |
+
Step 700/50462 | Loss: 4.2409 | LR: 4.67e-04 | 6991.7ms/step | 8.6 samples/s
|
| 262 |
+
Step 710/50462 | Loss: 4.2600 | LR: 4.73e-04 | 6990.2ms/step | 8.6 samples/s
|
| 263 |
+
Step 720/50462 | Loss: 4.2935 | LR: 4.80e-04 | 6996.9ms/step | 8.6 samples/s
|
| 264 |
+
Step 730/50462 | Loss: 4.2164 | LR: 4.87e-04 | 6997.2ms/step | 8.6 samples/s
|
| 265 |
+
Step 740/50462 | Loss: 4.1447 | LR: 4.93e-04 | 6999.5ms/step | 8.6 samples/s
|
| 266 |
+
Step 750/50462 | Loss: 4.1294 | LR: 5.00e-04 | 6998.5ms/step | 8.6 samples/s
|
| 267 |
+
Validation at step 750: Loss = 4.1136, Perplexity = 61.17
|
| 268 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-750
|
| 269 |
+
Step 760/50462 | Loss: 4.0937 | LR: 5.00e-04 | 10758.6ms/step | 5.6 samples/s
|
| 270 |
+
Step 770/50462 | Loss: 4.1263 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
|
| 271 |
+
Step 780/50462 | Loss: 4.0728 | LR: 5.00e-04 | 6989.5ms/step | 8.6 samples/s
|
| 272 |
+
Step 790/50462 | Loss: 4.0538 | LR: 5.00e-04 | 6994.4ms/step | 8.6 samples/s
|
| 273 |
+
Step 800/50462 | Loss: 4.0512 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
|
| 274 |
+
Step 810/50462 | Loss: 3.9448 | LR: 5.00e-04 | 6991.6ms/step | 8.6 samples/s
|
| 275 |
+
Step 820/50462 | Loss: 4.0090 | LR: 5.00e-04 | 6996.2ms/step | 8.6 samples/s
|
| 276 |
+
Step 830/50462 | Loss: 3.9644 | LR: 5.00e-04 | 6984.1ms/step | 8.6 samples/s
|
| 277 |
+
Step 840/50462 | Loss: 3.9260 | LR: 5.00e-04 | 6986.2ms/step | 8.6 samples/s
|
| 278 |
+
Step 850/50462 | Loss: 3.9136 | LR: 5.00e-04 | 6989.1ms/step | 8.6 samples/s
|
| 279 |
+
Step 860/50462 | Loss: 3.8627 | LR: 5.00e-04 | 6981.9ms/step | 8.6 samples/s
|
| 280 |
+
Step 870/50462 | Loss: 3.8631 | LR: 5.00e-04 | 6974.7ms/step | 8.6 samples/s
|
| 281 |
+
Step 880/50462 | Loss: 3.8533 | LR: 5.00e-04 | 6977.1ms/step | 8.6 samples/s
|
| 282 |
+
Step 890/50462 | Loss: 3.8499 | LR: 5.00e-04 | 6977.7ms/step | 8.6 samples/s
|
| 283 |
+
Step 900/50462 | Loss: 3.8223 | LR: 5.00e-04 | 6974.0ms/step | 8.6 samples/s
|
| 284 |
+
Step 910/50462 | Loss: 3.8381 | LR: 5.00e-04 | 6973.0ms/step | 8.6 samples/s
|
| 285 |
+
Step 920/50462 | Loss: 3.8187 | LR: 5.00e-04 | 6972.2ms/step | 8.6 samples/s
|
| 286 |
+
Step 930/50462 | Loss: 3.7817 | LR: 5.00e-04 | 6973.8ms/step | 8.6 samples/s
|
| 287 |
+
Step 940/50462 | Loss: 3.8183 | LR: 5.00e-04 | 6976.2ms/step | 8.6 samples/s
|
| 288 |
+
Step 950/50462 | Loss: 3.7734 | LR: 5.00e-04 | 6972.4ms/step | 8.6 samples/s
|
| 289 |
+
Step 960/50462 | Loss: 3.7546 | LR: 5.00e-04 | 6980.5ms/step | 8.6 samples/s
|
| 290 |
+
Step 970/50462 | Loss: 3.7664 | LR: 5.00e-04 | 6997.3ms/step | 8.6 samples/s
|
| 291 |
+
Step 980/50462 | Loss: 3.7394 | LR: 5.00e-04 | 7001.2ms/step | 8.6 samples/s
|
| 292 |
+
Step 990/50462 | Loss: 3.7606 | LR: 5.00e-04 | 7003.0ms/step | 8.6 samples/s
|
| 293 |
+
Step 1000/50462 | Loss: 3.7216 | LR: 5.00e-04 | 7003.4ms/step | 8.6 samples/s
|
| 294 |
+
Validation at step 1000: Loss = 3.6926, Perplexity = 40.15
|
| 295 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1000
|
| 296 |
+
Step 1010/50462 | Loss: 3.6989 | LR: 5.00e-04 | 10784.3ms/step | 5.6 samples/s
|
| 297 |
+
Step 1020/50462 | Loss: 3.6964 | LR: 5.00e-04 | 7000.7ms/step | 8.6 samples/s
|
| 298 |
+
Step 1030/50462 | Loss: 3.6747 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
|
| 299 |
+
Step 1040/50462 | Loss: 3.6733 | LR: 5.00e-04 | 6981.7ms/step | 8.6 samples/s
|
| 300 |
+
Step 1050/50462 | Loss: 3.6548 | LR: 5.00e-04 | 6976.3ms/step | 8.6 samples/s
|
| 301 |
+
Step 1060/50462 | Loss: 3.6781 | LR: 5.00e-04 | 6978.6ms/step | 8.6 samples/s
|
| 302 |
+
Step 1070/50462 | Loss: 3.6458 | LR: 5.00e-04 | 6984.2ms/step | 8.6 samples/s
|
| 303 |
+
Step 1080/50462 | Loss: 3.6464 | LR: 5.00e-04 | 6995.5ms/step | 8.6 samples/s
|
| 304 |
+
Step 1090/50462 | Loss: 3.6192 | LR: 5.00e-04 | 6999.7ms/step | 8.6 samples/s
|
| 305 |
+
Step 1100/50462 | Loss: 3.6117 | LR: 5.00e-04 | 6995.7ms/step | 8.6 samples/s
|
| 306 |
+
Step 1110/50462 | Loss: 3.6503 | LR: 5.00e-04 | 6990.1ms/step | 8.6 samples/s
|
| 307 |
+
Step 1120/50462 | Loss: 3.5846 | LR: 5.00e-04 | 6988.8ms/step | 8.6 samples/s
|
| 308 |
+
Step 1130/50462 | Loss: 3.5687 | LR: 5.00e-04 | 6988.1ms/step | 8.6 samples/s
|
| 309 |
+
Step 1140/50462 | Loss: 3.5888 | LR: 5.00e-04 | 6989.8ms/step | 8.6 samples/s
|
| 310 |
+
Step 1150/50462 | Loss: 3.5869 | LR: 5.00e-04 | 6985.9ms/step | 8.6 samples/s
|
| 311 |
+
Step 1160/50462 | Loss: 3.5741 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
|
| 312 |
+
Step 1170/50462 | Loss: 3.5614 | LR: 5.00e-04 | 6986.4ms/step | 8.6 samples/s
|
| 313 |
+
Step 1180/50462 | Loss: 3.5469 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
|
| 314 |
+
Step 1190/50462 | Loss: 3.5895 | LR: 5.00e-04 | 6986.5ms/step | 8.6 samples/s
|
| 315 |
+
Step 1200/50462 | Loss: 3.4897 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
|
| 316 |
+
Step 1210/50462 | Loss: 3.5457 | LR: 5.00e-04 | 6985.7ms/step | 8.6 samples/s
|
| 317 |
+
Step 1220/50462 | Loss: 3.5586 | LR: 5.00e-04 | 6985.5ms/step | 8.6 samples/s
|
| 318 |
+
Step 1230/50462 | Loss: 3.5576 | LR: 5.00e-04 | 6987.6ms/step | 8.6 samples/s
|
| 319 |
+
Step 1240/50462 | Loss: 3.5320 | LR: 5.00e-04 | 6987.8ms/step | 8.6 samples/s
|
| 320 |
+
Step 1250/50462 | Loss: 3.5228 | LR: 5.00e-04 | 6986.3ms/step | 8.6 samples/s
|
| 321 |
+
Validation at step 1250: Loss = 3.4897, Perplexity = 32.78
|
| 322 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1250
|
| 323 |
+
Step 1260/50462 | Loss: 3.4950 | LR: 5.00e-04 | 10757.8ms/step | 5.6 samples/s
|
| 324 |
+
Step 1270/50462 | Loss: 3.4530 | LR: 5.00e-04 | 6984.5ms/step | 8.6 samples/s
|
| 325 |
+
Step 1280/50462 | Loss: 3.5241 | LR: 5.00e-04 | 6988.9ms/step | 8.6 samples/s
|
| 326 |
+
Step 1290/50462 | Loss: 3.4532 | LR: 5.00e-04 | 6986.1ms/step | 8.6 samples/s
|
| 327 |
+
Step 1300/50462 | Loss: 3.4954 | LR: 5.00e-04 | 6988.0ms/step | 8.6 samples/s
|
| 328 |
+
Step 1310/50462 | Loss: 3.5101 | LR: 5.00e-04 | 6986.1ms/step | 8.6 samples/s
|
| 329 |
+
Step 1320/50462 | Loss: 3.4786 | LR: 5.00e-04 | 6989.8ms/step | 8.6 samples/s
|
| 330 |
+
Step 1330/50462 | Loss: 3.4448 | LR: 5.00e-04 | 6984.2ms/step | 8.6 samples/s
|
| 331 |
+
Step 1340/50462 | Loss: 3.4296 | LR: 5.00e-04 | 6989.5ms/step | 8.6 samples/s
|
| 332 |
+
Step 1350/50462 | Loss: 3.4514 | LR: 5.00e-04 | 6988.0ms/step | 8.6 samples/s
|
| 333 |
+
Step 1360/50462 | Loss: 3.4206 | LR: 5.00e-04 | 6985.8ms/step | 8.6 samples/s
|
| 334 |
+
Step 1370/50462 | Loss: 3.4638 | LR: 5.00e-04 | 6990.2ms/step | 8.6 samples/s
|
| 335 |
+
Step 1380/50462 | Loss: 3.4280 | LR: 5.00e-04 | 6986.4ms/step | 8.6 samples/s
|
| 336 |
+
Step 1390/50462 | Loss: 3.3971 | LR: 5.00e-04 | 6987.5ms/step | 8.6 samples/s
|
| 337 |
+
Step 1400/50462 | Loss: 3.4146 | LR: 5.00e-04 | 6987.4ms/step | 8.6 samples/s
|
| 338 |
+
Step 1410/50462 | Loss: 3.4468 | LR: 5.00e-04 | 6980.2ms/step | 8.6 samples/s
|
| 339 |
+
Step 1420/50462 | Loss: 3.4424 | LR: 5.00e-04 | 6979.3ms/step | 8.6 samples/s
|
| 340 |
+
Step 1430/50462 | Loss: 3.4211 | LR: 5.00e-04 | 6980.1ms/step | 8.6 samples/s
|
| 341 |
+
Step 1440/50462 | Loss: 3.4276 | LR: 5.00e-04 | 6978.7ms/step | 8.6 samples/s
|
| 342 |
+
Step 1450/50462 | Loss: 3.3767 | LR: 5.00e-04 | 6978.1ms/step | 8.6 samples/s
|
| 343 |
+
Step 1460/50462 | Loss: 3.3895 | LR: 5.00e-04 | 6975.8ms/step | 8.6 samples/s
|
| 344 |
+
Step 1470/50462 | Loss: 3.4021 | LR: 5.00e-04 | 6977.1ms/step | 8.6 samples/s
|
| 345 |
+
Step 1480/50462 | Loss: 3.3726 | LR: 5.00e-04 | 6975.9ms/step | 8.6 samples/s
|
| 346 |
+
Step 1490/50462 | Loss: 3.4028 | LR: 5.00e-04 | 6975.7ms/step | 8.6 samples/s
|
| 347 |
+
Step 1500/50462 | Loss: 3.3659 | LR: 5.00e-04 | 6976.2ms/step | 8.6 samples/s
|
| 348 |
+
Validation at step 1500: Loss = 3.3674, Perplexity = 29.00
|
| 349 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1500
|
| 350 |
+
Step 1510/50462 | Loss: 3.3810 | LR: 5.00e-04 | 10759.7ms/step | 5.6 samples/s
|
| 351 |
+
Step 1520/50462 | Loss: 3.3442 | LR: 5.00e-04 | 7000.4ms/step | 8.6 samples/s
|
| 352 |
+
Step 1530/50462 | Loss: 3.3526 | LR: 5.00e-04 | 7016.7ms/step | 8.6 samples/s
|
| 353 |
+
Step 1540/50462 | Loss: 3.3672 | LR: 5.00e-04 | 7020.0ms/step | 8.5 samples/s
|
| 354 |
+
Step 1550/50462 | Loss: 3.3788 | LR: 5.00e-04 | 7018.8ms/step | 8.5 samples/s
|
| 355 |
+
Step 1560/50462 | Loss: 3.3520 | LR: 5.00e-04 | 7002.4ms/step | 8.6 samples/s
|
| 356 |
+
Step 1570/50462 | Loss: 3.3618 | LR: 5.00e-04 | 7009.3ms/step | 8.6 samples/s
|
| 357 |
+
Step 1580/50462 | Loss: 3.3222 | LR: 5.00e-04 | 7015.6ms/step | 8.6 samples/s
|
| 358 |
+
Step 1590/50462 | Loss: 3.3415 | LR: 5.00e-04 | 6992.0ms/step | 8.6 samples/s
|
| 359 |
+
Step 1600/50462 | Loss: 3.3620 | LR: 5.00e-04 | 6991.1ms/step | 8.6 samples/s
|
| 360 |
+
Step 1610/50462 | Loss: 3.3402 | LR: 5.00e-04 | 6984.7ms/step | 8.6 samples/s
|
| 361 |
+
Step 1620/50462 | Loss: 3.3494 | LR: 5.00e-04 | 6991.9ms/step | 8.6 samples/s
|
| 362 |
+
Step 1630/50462 | Loss: 3.3329 | LR: 5.00e-04 | 7008.2ms/step | 8.6 samples/s
|
| 363 |
+
Step 1640/50462 | Loss: 3.3089 | LR: 5.00e-04 | 7019.5ms/step | 8.5 samples/s
|
| 364 |
+
Step 1650/50462 | Loss: 3.3211 | LR: 5.00e-04 | 7009.9ms/step | 8.6 samples/s
|
| 365 |
+
Step 1660/50462 | Loss: 3.3309 | LR: 5.00e-04 | 7007.8ms/step | 8.6 samples/s
|
| 366 |
+
Step 1670/50462 | Loss: 3.2594 | LR: 5.00e-04 | 7007.3ms/step | 8.6 samples/s
|
| 367 |
+
Step 1680/50462 | Loss: 3.3242 | LR: 5.00e-04 | 7004.2ms/step | 8.6 samples/s
|
| 368 |
+
Step 1690/50462 | Loss: 3.3271 | LR: 5.00e-04 | 6994.7ms/step | 8.6 samples/s
|
| 369 |
+
Step 1700/50462 | Loss: 3.2683 | LR: 5.00e-04 | 6996.2ms/step | 8.6 samples/s
|
| 370 |
+
Step 1710/50462 | Loss: 3.2889 | LR: 5.00e-04 | 6994.8ms/step | 8.6 samples/s
|
| 371 |
+
Step 1720/50462 | Loss: 3.2799 | LR: 5.00e-04 | 6991.3ms/step | 8.6 samples/s
|
| 372 |
+
Step 1730/50462 | Loss: 3.2937 | LR: 5.00e-04 | 6992.7ms/step | 8.6 samples/s
|
| 373 |
+
Step 1740/50462 | Loss: 3.2602 | LR: 5.00e-04 | 6992.2ms/step | 8.6 samples/s
|
| 374 |
+
Step 1750/50462 | Loss: 3.2789 | LR: 5.00e-04 | 6999.4ms/step | 8.6 samples/s
|
| 375 |
+
Validation at step 1750: Loss = 3.2792, Perplexity = 26.55
|
| 376 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-1750
|
| 377 |
+
Step 1760/50462 | Loss: 3.2848 | LR: 4.99e-04 | 10795.4ms/step | 5.6 samples/s
|
| 378 |
+
Step 1770/50462 | Loss: 3.2934 | LR: 4.99e-04 | 6996.0ms/step | 8.6 samples/s
|
| 379 |
+
Step 1780/50462 | Loss: 3.2656 | LR: 4.99e-04 | 7001.8ms/step | 8.6 samples/s
|
| 380 |
+
Step 1790/50462 | Loss: 3.2855 | LR: 4.99e-04 | 7008.4ms/step | 8.6 samples/s
|
| 381 |
+
Step 1800/50462 | Loss: 3.2852 | LR: 4.99e-04 | 7007.4ms/step | 8.6 samples/s
|
| 382 |
+
Step 1810/50462 | Loss: 3.3031 | LR: 4.99e-04 | 7002.4ms/step | 8.6 samples/s
|
| 383 |
+
Step 1820/50462 | Loss: 3.2804 | LR: 4.99e-04 | 7004.9ms/step | 8.6 samples/s
|
| 384 |
+
Step 1830/50462 | Loss: 3.2724 | LR: 4.99e-04 | 7004.9ms/step | 8.6 samples/s
|
| 385 |
+
Step 1840/50462 | Loss: 3.2435 | LR: 4.99e-04 | 6999.7ms/step | 8.6 samples/s
|
| 386 |
+
Step 1850/50462 | Loss: 3.2467 | LR: 4.99e-04 | 6999.1ms/step | 8.6 samples/s
|
| 387 |
+
Step 1860/50462 | Loss: 3.2667 | LR: 4.99e-04 | 7000.4ms/step | 8.6 samples/s
|
| 388 |
+
Step 1870/50462 | Loss: 3.2725 | LR: 4.99e-04 | 7002.4ms/step | 8.6 samples/s
|
| 389 |
+
Step 1880/50462 | Loss: 3.2516 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
|
| 390 |
+
Step 1890/50462 | Loss: 3.2262 | LR: 4.99e-04 | 7006.3ms/step | 8.6 samples/s
|
| 391 |
+
Step 1900/50462 | Loss: 3.2665 | LR: 4.99e-04 | 7008.0ms/step | 8.6 samples/s
|
| 392 |
+
Step 1910/50462 | Loss: 3.2557 | LR: 4.99e-04 | 6999.6ms/step | 8.6 samples/s
|
| 393 |
+
Step 1920/50462 | Loss: 3.2344 | LR: 4.99e-04 | 7007.4ms/step | 8.6 samples/s
|
| 394 |
+
Step 1930/50462 | Loss: 3.2331 | LR: 4.99e-04 | 7006.3ms/step | 8.6 samples/s
|
| 395 |
+
Step 1940/50462 | Loss: 3.2540 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
|
| 396 |
+
Step 1950/50462 | Loss: 3.2642 | LR: 4.99e-04 | 7003.3ms/step | 8.6 samples/s
|
| 397 |
+
Step 1960/50462 | Loss: 3.2328 | LR: 4.99e-04 | 7001.2ms/step | 8.6 samples/s
|
| 398 |
+
Step 1970/50462 | Loss: 3.2330 | LR: 4.99e-04 | 6995.3ms/step | 8.6 samples/s
|
| 399 |
+
Step 1980/50462 | Loss: 3.2341 | LR: 4.99e-04 | 6988.7ms/step | 8.6 samples/s
|
| 400 |
+
Step 1990/50462 | Loss: 3.2301 | LR: 4.99e-04 | 6986.6ms/step | 8.6 samples/s
|
| 401 |
+
Step 2000/50462 | Loss: 3.2461 | LR: 4.99e-04 | 6986.6ms/step | 8.6 samples/s
|
| 402 |
+
Validation at step 2000: Loss = 3.2130, Perplexity = 24.85
|
| 403 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2000
|
| 404 |
+
Step 2010/50462 | Loss: 3.1880 | LR: 4.99e-04 | 10783.7ms/step | 5.6 samples/s
|
| 405 |
+
Step 2020/50462 | Loss: 3.2372 | LR: 4.99e-04 | 6986.3ms/step | 8.6 samples/s
|
| 406 |
+
Step 2030/50462 | Loss: 3.2083 | LR: 4.99e-04 | 6990.2ms/step | 8.6 samples/s
|
| 407 |
+
Step 2040/50462 | Loss: 3.2101 | LR: 4.99e-04 | 6988.6ms/step | 8.6 samples/s
|
| 408 |
+
Step 2050/50462 | Loss: 3.1618 | LR: 4.99e-04 | 6982.2ms/step | 8.6 samples/s
|
| 409 |
+
Step 2060/50462 | Loss: 3.2344 | LR: 4.99e-04 | 6984.4ms/step | 8.6 samples/s
|
| 410 |
+
Step 2070/50462 | Loss: 3.2079 | LR: 4.99e-04 | 7000.0ms/step | 8.6 samples/s
|
| 411 |
+
Step 2080/50462 | Loss: 3.2033 | LR: 4.99e-04 | 7013.4ms/step | 8.6 samples/s
|
| 412 |
+
Step 2090/50462 | Loss: 3.2283 | LR: 4.99e-04 | 7008.2ms/step | 8.6 samples/s
|
| 413 |
+
Step 2100/50462 | Loss: 3.2070 | LR: 4.99e-04 | 7010.9ms/step | 8.6 samples/s
|
| 414 |
+
Step 2110/50462 | Loss: 3.2140 | LR: 4.99e-04 | 7016.3ms/step | 8.6 samples/s
|
| 415 |
+
Step 2120/50462 | Loss: 3.2027 | LR: 4.99e-04 | 7006.6ms/step | 8.6 samples/s
|
| 416 |
+
Step 2130/50462 | Loss: 3.2023 | LR: 4.99e-04 | 7015.1ms/step | 8.6 samples/s
|
| 417 |
+
Step 2140/50462 | Loss: 3.2078 | LR: 4.99e-04 | 7003.2ms/step | 8.6 samples/s
|
| 418 |
+
Step 2150/50462 | Loss: 3.1806 | LR: 4.99e-04 | 6993.1ms/step | 8.6 samples/s
|
| 419 |
+
Step 2160/50462 | Loss: 3.2147 | LR: 4.99e-04 | 6990.6ms/step | 8.6 samples/s
|
| 420 |
+
Step 2170/50462 | Loss: 3.1855 | LR: 4.99e-04 | 6987.0ms/step | 8.6 samples/s
|
| 421 |
+
Step 2180/50462 | Loss: 3.2051 | LR: 4.99e-04 | 6994.4ms/step | 8.6 samples/s
|
| 422 |
+
Step 2190/50462 | Loss: 3.1860 | LR: 4.99e-04 | 7002.5ms/step | 8.6 samples/s
|
| 423 |
+
Step 2200/50462 | Loss: 3.1900 | LR: 4.99e-04 | 7005.0ms/step | 8.6 samples/s
|
| 424 |
+
Step 2210/50462 | Loss: 3.1696 | LR: 4.99e-04 | 6998.2ms/step | 8.6 samples/s
|
| 425 |
+
Step 2220/50462 | Loss: 3.2034 | LR: 4.99e-04 | 6996.9ms/step | 8.6 samples/s
|
| 426 |
+
Step 2230/50462 | Loss: 3.1856 | LR: 4.99e-04 | 6994.2ms/step | 8.6 samples/s
|
| 427 |
+
Step 2240/50462 | Loss: 3.1857 | LR: 4.99e-04 | 6994.2ms/step | 8.6 samples/s
|
| 428 |
+
Step 2250/50462 | Loss: 3.1901 | LR: 4.99e-04 | 6991.9ms/step | 8.6 samples/s
|
| 429 |
+
Validation at step 2250: Loss = 3.1624, Perplexity = 23.63
|
| 430 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2250
|
| 431 |
+
Step 2260/50462 | Loss: 3.1717 | LR: 4.99e-04 | 10763.1ms/step | 5.6 samples/s
|
| 432 |
+
Step 2270/50462 | Loss: 3.1833 | LR: 4.99e-04 | 6996.3ms/step | 8.6 samples/s
|
| 433 |
+
Step 2280/50462 | Loss: 3.1842 | LR: 4.99e-04 | 6993.1ms/step | 8.6 samples/s
|
| 434 |
+
Step 2290/50462 | Loss: 3.1703 | LR: 4.99e-04 | 6994.1ms/step | 8.6 samples/s
|
| 435 |
+
Step 2300/50462 | Loss: 3.1720 | LR: 4.99e-04 | 6991.5ms/step | 8.6 samples/s
|
| 436 |
+
Step 2310/50462 | Loss: 3.1915 | LR: 4.99e-04 | 6993.6ms/step | 8.6 samples/s
|
| 437 |
+
Step 2320/50462 | Loss: 3.1855 | LR: 4.99e-04 | 6995.1ms/step | 8.6 samples/s
|
| 438 |
+
Step 2330/50462 | Loss: 3.1540 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
|
| 439 |
+
Step 2340/50462 | Loss: 3.1747 | LR: 4.99e-04 | 7005.2ms/step | 8.6 samples/s
|
| 440 |
+
Step 2350/50462 | Loss: 3.1328 | LR: 4.99e-04 | 7004.4ms/step | 8.6 samples/s
|
| 441 |
+
Step 2360/50462 | Loss: 3.1299 | LR: 4.99e-04 | 7000.5ms/step | 8.6 samples/s
|
| 442 |
+
Step 2370/50462 | Loss: 3.1529 | LR: 4.99e-04 | 6998.4ms/step | 8.6 samples/s
|
| 443 |
+
Step 2380/50462 | Loss: 3.1293 | LR: 4.99e-04 | 6999.3ms/step | 8.6 samples/s
|
| 444 |
+
Step 2390/50462 | Loss: 3.1752 | LR: 4.99e-04 | 6999.6ms/step | 8.6 samples/s
|
| 445 |
+
Step 2400/50462 | Loss: 3.1625 | LR: 4.99e-04 | 6996.9ms/step | 8.6 samples/s
|
| 446 |
+
Step 2410/50462 | Loss: 3.1279 | LR: 4.99e-04 | 7000.3ms/step | 8.6 samples/s
|
| 447 |
+
Step 2420/50462 | Loss: 3.1246 | LR: 4.99e-04 | 7003.3ms/step | 8.6 samples/s
|
| 448 |
+
Step 2430/50462 | Loss: 3.1375 | LR: 4.99e-04 | 7000.2ms/step | 8.6 samples/s
|
| 449 |
+
Step 2440/50462 | Loss: 3.1167 | LR: 4.99e-04 | 7009.0ms/step | 8.6 samples/s
|
| 450 |
+
Step 2450/50462 | Loss: 3.1839 | LR: 4.99e-04 | 7001.4ms/step | 8.6 samples/s
|
| 451 |
+
Step 2460/50462 | Loss: 3.1271 | LR: 4.99e-04 | 6998.0ms/step | 8.6 samples/s
|
| 452 |
+
Step 2470/50462 | Loss: 3.1414 | LR: 4.99e-04 | 7006.0ms/step | 8.6 samples/s
|
| 453 |
+
Step 2480/50462 | Loss: 3.1352 | LR: 4.99e-04 | 7005.4ms/step | 8.6 samples/s
|
| 454 |
+
Step 2490/50462 | Loss: 3.1253 | LR: 4.98e-04 | 7004.1ms/step | 8.6 samples/s
|
| 455 |
+
Step 2500/50462 | Loss: 3.1194 | LR: 4.98e-04 | 7005.5ms/step | 8.6 samples/s
|
| 456 |
+
Validation at step 2500: Loss = 3.1202, Perplexity = 22.65
|
| 457 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2500
|
| 458 |
+
Step 2510/50462 | Loss: 3.1033 | LR: 4.98e-04 | 10794.6ms/step | 5.6 samples/s
|
| 459 |
+
Step 2520/50462 | Loss: 3.1173 | LR: 4.98e-04 | 6997.5ms/step | 8.6 samples/s
|
| 460 |
+
Step 2530/50462 | Loss: 3.1184 | LR: 4.98e-04 | 6997.1ms/step | 8.6 samples/s
|
| 461 |
+
Step 2540/50462 | Loss: 3.1084 | LR: 4.98e-04 | 6996.1ms/step | 8.6 samples/s
|
| 462 |
+
Step 2550/50462 | Loss: 3.1290 | LR: 4.98e-04 | 6995.3ms/step | 8.6 samples/s
|
| 463 |
+
Step 2560/50462 | Loss: 3.1058 | LR: 4.98e-04 | 6989.3ms/step | 8.6 samples/s
|
| 464 |
+
Step 2570/50462 | Loss: 3.1115 | LR: 4.98e-04 | 6985.1ms/step | 8.6 samples/s
|
| 465 |
+
Step 2580/50462 | Loss: 3.0768 | LR: 4.98e-04 | 6987.8ms/step | 8.6 samples/s
|
| 466 |
+
Step 2590/50462 | Loss: 3.0940 | LR: 4.98e-04 | 6995.2ms/step | 8.6 samples/s
|
| 467 |
+
Step 2600/50462 | Loss: 3.1402 | LR: 4.98e-04 | 6995.5ms/step | 8.6 samples/s
|
| 468 |
+
Step 2610/50462 | Loss: 3.1374 | LR: 4.98e-04 | 6993.3ms/step | 8.6 samples/s
|
| 469 |
+
Step 2620/50462 | Loss: 3.1452 | LR: 4.98e-04 | 6999.2ms/step | 8.6 samples/s
|
| 470 |
+
Step 2630/50462 | Loss: 3.0915 | LR: 4.98e-04 | 6996.9ms/step | 8.6 samples/s
|
| 471 |
+
Step 2640/50462 | Loss: 3.1067 | LR: 4.98e-04 | 6995.4ms/step | 8.6 samples/s
|
| 472 |
+
Step 2650/50462 | Loss: 3.1091 | LR: 4.98e-04 | 7000.9ms/step | 8.6 samples/s
|
| 473 |
+
Step 2660/50462 | Loss: 3.1264 | LR: 4.98e-04 | 7014.9ms/step | 8.6 samples/s
|
| 474 |
+
Step 2670/50462 | Loss: 3.1062 | LR: 4.98e-04 | 7016.7ms/step | 8.6 samples/s
|
| 475 |
+
Step 2680/50462 | Loss: 3.1093 | LR: 4.98e-04 | 7029.1ms/step | 8.5 samples/s
|
| 476 |
+
Step 2690/50462 | Loss: 3.0895 | LR: 4.98e-04 | 7018.9ms/step | 8.5 samples/s
|
| 477 |
+
Step 2700/50462 | Loss: 3.1212 | LR: 4.98e-04 | 7019.5ms/step | 8.5 samples/s
|
| 478 |
+
Step 2710/50462 | Loss: 3.1282 | LR: 4.98e-04 | 7016.4ms/step | 8.6 samples/s
|
| 479 |
+
Step 2720/50462 | Loss: 3.1085 | LR: 4.98e-04 | 7016.2ms/step | 8.6 samples/s
|
| 480 |
+
Step 2730/50462 | Loss: 3.1022 | LR: 4.98e-04 | 7005.8ms/step | 8.6 samples/s
|
| 481 |
+
Step 2740/50462 | Loss: 3.1090 | LR: 4.98e-04 | 6988.8ms/step | 8.6 samples/s
|
| 482 |
+
Step 2750/50462 | Loss: 3.1129 | LR: 4.98e-04 | 6995.2ms/step | 8.6 samples/s
|
| 483 |
+
Validation at step 2750: Loss = 3.0843, Perplexity = 21.85
|
| 484 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-2750
|
| 485 |
+
Step 2760/50462 | Loss: 3.1199 | LR: 4.98e-04 | 10776.6ms/step | 5.6 samples/s
|
| 486 |
+
Step 2770/50462 | Loss: 3.1243 | LR: 4.98e-04 | 7010.9ms/step | 8.6 samples/s
|
| 487 |
+
Step 2780/50462 | Loss: 3.1189 | LR: 4.98e-04 | 7020.1ms/step | 8.5 samples/s
|
| 488 |
+
Step 2790/50462 | Loss: 3.1252 | LR: 4.98e-04 | 7017.2ms/step | 8.6 samples/s
|
| 489 |
+
Step 2800/50462 | Loss: 3.0913 | LR: 4.98e-04 | 7014.1ms/step | 8.6 samples/s
|
| 490 |
+
Step 2810/50462 | Loss: 3.1074 | LR: 4.98e-04 | 7006.9ms/step | 8.6 samples/s
|
| 491 |
+
Step 2820/50462 | Loss: 3.0836 | LR: 4.98e-04 | 7007.0ms/step | 8.6 samples/s
|
| 492 |
+
Step 2830/50462 | Loss: 3.0726 | LR: 4.98e-04 | 7006.4ms/step | 8.6 samples/s
|
| 493 |
+
Step 2840/50462 | Loss: 3.0717 | LR: 4.98e-04 | 7008.0ms/step | 8.6 samples/s
|
| 494 |
+
Step 2850/50462 | Loss: 3.0960 | LR: 4.98e-04 | 7006.4ms/step | 8.6 samples/s
|
| 495 |
+
Step 2860/50462 | Loss: 3.0877 | LR: 4.98e-04 | 7006.9ms/step | 8.6 samples/s
|
| 496 |
+
Step 2870/50462 | Loss: 3.0788 | LR: 4.98e-04 | 7007.4ms/step | 8.6 samples/s
|
| 497 |
+
Step 2880/50462 | Loss: 3.0712 | LR: 4.98e-04 | 7008.6ms/step | 8.6 samples/s
|
| 498 |
+
Step 2890/50462 | Loss: 3.0741 | LR: 4.98e-04 | 7005.6ms/step | 8.6 samples/s
|
| 499 |
+
Step 2900/50462 | Loss: 3.1151 | LR: 4.98e-04 | 7003.6ms/step | 8.6 samples/s
|
| 500 |
+
Step 2910/50462 | Loss: 3.0641 | LR: 4.98e-04 | 7006.8ms/step | 8.6 samples/s
|
| 501 |
+
Step 2920/50462 | Loss: 3.0694 | LR: 4.98e-04 | 7006.8ms/step | 8.6 samples/s
|
| 502 |
+
Step 2930/50462 | Loss: 3.0819 | LR: 4.98e-04 | 7009.3ms/step | 8.6 samples/s
|
| 503 |
+
Step 2940/50462 | Loss: 3.0491 | LR: 4.98e-04 | 7006.2ms/step | 8.6 samples/s
|
| 504 |
+
Step 2950/50462 | Loss: 3.0619 | LR: 4.98e-04 | 7012.4ms/step | 8.6 samples/s
|
| 505 |
+
Step 2960/50462 | Loss: 3.0773 | LR: 4.98e-04 | 7011.6ms/step | 8.6 samples/s
|
| 506 |
+
Step 2970/50462 | Loss: 3.0798 | LR: 4.98e-04 | 7007.0ms/step | 8.6 samples/s
|
| 507 |
+
Step 2980/50462 | Loss: 3.0732 | LR: 4.98e-04 | 7011.1ms/step | 8.6 samples/s
|
| 508 |
+
Step 2990/50462 | Loss: 3.0639 | LR: 4.97e-04 | 7007.3ms/step | 8.6 samples/s
|
| 509 |
+
Step 3000/50462 | Loss: 3.0502 | LR: 4.97e-04 | 7009.6ms/step | 8.6 samples/s
|
| 510 |
+
Validation at step 3000: Loss = 3.0547, Perplexity = 21.22
|
| 511 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3000
|
| 512 |
+
Step 3010/50462 | Loss: 3.0558 | LR: 4.97e-04 | 10794.4ms/step | 5.6 samples/s
|
| 513 |
+
Step 3020/50462 | Loss: 3.0383 | LR: 4.97e-04 | 7004.8ms/step | 8.6 samples/s
|
| 514 |
+
Step 3030/50462 | Loss: 3.0735 | LR: 4.97e-04 | 7006.4ms/step | 8.6 samples/s
|
| 515 |
+
Step 3040/50462 | Loss: 3.0494 | LR: 4.97e-04 | 7010.6ms/step | 8.6 samples/s
|
| 516 |
+
Step 3050/50462 | Loss: 3.0584 | LR: 4.97e-04 | 7009.5ms/step | 8.6 samples/s
|
| 517 |
+
Step 3060/50462 | Loss: 3.0662 | LR: 4.97e-04 | 7012.8ms/step | 8.6 samples/s
|
| 518 |
+
Step 3070/50462 | Loss: 3.0913 | LR: 4.97e-04 | 7011.4ms/step | 8.6 samples/s
|
| 519 |
+
Step 3080/50462 | Loss: 3.0764 | LR: 4.97e-04 | 7006.5ms/step | 8.6 samples/s
|
| 520 |
+
Step 3090/50462 | Loss: 3.0916 | LR: 4.97e-04 | 7009.7ms/step | 8.6 samples/s
|
| 521 |
+
Step 3100/50462 | Loss: 3.0698 | LR: 4.97e-04 | 7011.2ms/step | 8.6 samples/s
|
| 522 |
+
Step 3110/50462 | Loss: 3.0645 | LR: 4.97e-04 | 7010.5ms/step | 8.6 samples/s
|
| 523 |
+
Step 3120/50462 | Loss: 3.0299 | LR: 4.97e-04 | 7008.8ms/step | 8.6 samples/s
|
| 524 |
+
Step 3130/50462 | Loss: 3.0276 | LR: 4.97e-04 | 7005.5ms/step | 8.6 samples/s
|
| 525 |
+
Step 3140/50462 | Loss: 3.0592 | LR: 4.97e-04 | 7006.3ms/step | 8.6 samples/s
|
| 526 |
+
Step 3150/50462 | Loss: 3.0562 | LR: 4.97e-04 | 7005.0ms/step | 8.6 samples/s
|
| 527 |
+
Step 3160/50462 | Loss: 3.0228 | LR: 4.97e-04 | 7003.1ms/step | 8.6 samples/s
|
| 528 |
+
Step 3170/50462 | Loss: 3.0266 | LR: 4.97e-04 | 7005.8ms/step | 8.6 samples/s
|
| 529 |
+
Step 3180/50462 | Loss: 3.0595 | LR: 4.97e-04 | 7001.5ms/step | 8.6 samples/s
|
| 530 |
+
Step 3190/50462 | Loss: 3.0480 | LR: 4.97e-04 | 6990.3ms/step | 8.6 samples/s
|
| 531 |
+
Step 3200/50462 | Loss: 3.0290 | LR: 4.97e-04 | 6992.9ms/step | 8.6 samples/s
|
| 532 |
+
Step 3210/50462 | Loss: 3.0576 | LR: 4.97e-04 | 6991.4ms/step | 8.6 samples/s
|
| 533 |
+
Step 3220/50462 | Loss: 3.0312 | LR: 4.97e-04 | 6986.6ms/step | 8.6 samples/s
|
| 534 |
+
Step 3230/50462 | Loss: 3.0388 | LR: 4.97e-04 | 6985.9ms/step | 8.6 samples/s
|
| 535 |
+
Step 3240/50462 | Loss: 3.0631 | LR: 4.97e-04 | 6986.2ms/step | 8.6 samples/s
|
| 536 |
+
Step 3250/50462 | Loss: 3.0602 | LR: 4.97e-04 | 6995.1ms/step | 8.6 samples/s
|
| 537 |
+
Validation at step 3250: Loss = 3.0245, Perplexity = 20.58
|
| 538 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3250
|
| 539 |
+
Step 3260/50462 | Loss: 3.0145 | LR: 4.97e-04 | 10809.9ms/step | 5.6 samples/s
|
| 540 |
+
Step 3270/50462 | Loss: 3.0456 | LR: 4.97e-04 | 7006.2ms/step | 8.6 samples/s
|
| 541 |
+
Step 3280/50462 | Loss: 3.0094 | LR: 4.97e-04 | 7012.6ms/step | 8.6 samples/s
|
| 542 |
+
Step 3290/50462 | Loss: 3.0381 | LR: 4.97e-04 | 7013.9ms/step | 8.6 samples/s
|
| 543 |
+
Step 3300/50462 | Loss: 3.0133 | LR: 4.97e-04 | 7018.6ms/step | 8.5 samples/s
|
| 544 |
+
Step 3310/50462 | Loss: 3.0206 | LR: 4.97e-04 | 7014.5ms/step | 8.6 samples/s
|
| 545 |
+
Step 3320/50462 | Loss: 3.0257 | LR: 4.97e-04 | 7006.3ms/step | 8.6 samples/s
|
| 546 |
+
Step 3330/50462 | Loss: 3.0168 | LR: 4.97e-04 | 7018.8ms/step | 8.5 samples/s
|
| 547 |
+
Step 3340/50462 | Loss: 3.0034 | LR: 4.97e-04 | 7012.0ms/step | 8.6 samples/s
|
| 548 |
+
Step 3350/50462 | Loss: 3.0155 | LR: 4.97e-04 | 6993.5ms/step | 8.6 samples/s
|
| 549 |
+
Step 3360/50462 | Loss: 3.0274 | LR: 4.97e-04 | 6995.4ms/step | 8.6 samples/s
|
| 550 |
+
Step 3370/50462 | Loss: 3.0142 | LR: 4.97e-04 | 6987.6ms/step | 8.6 samples/s
|
| 551 |
+
Step 3380/50462 | Loss: 3.0444 | LR: 4.97e-04 | 6988.8ms/step | 8.6 samples/s
|
| 552 |
+
Step 3390/50462 | Loss: 3.0144 | LR: 4.97e-04 | 7003.5ms/step | 8.6 samples/s
|
| 553 |
+
Step 3400/50462 | Loss: 3.0029 | LR: 4.97e-04 | 7014.1ms/step | 8.6 samples/s
|
| 554 |
+
Step 3410/50462 | Loss: 3.0591 | LR: 4.96e-04 | 7007.0ms/step | 8.6 samples/s
|
| 555 |
+
Step 3420/50462 | Loss: 3.0262 | LR: 4.96e-04 | 7005.3ms/step | 8.6 samples/s
|
| 556 |
+
Step 3430/50462 | Loss: 3.0395 | LR: 4.96e-04 | 7007.2ms/step | 8.6 samples/s
|
| 557 |
+
Step 3440/50462 | Loss: 3.0176 | LR: 4.96e-04 | 7003.7ms/step | 8.6 samples/s
|
| 558 |
+
Step 3450/50462 | Loss: 2.9979 | LR: 4.96e-04 | 7002.4ms/step | 8.6 samples/s
|
| 559 |
+
Step 3460/50462 | Loss: 3.0453 | LR: 4.96e-04 | 7002.0ms/step | 8.6 samples/s
|
| 560 |
+
Step 3470/50462 | Loss: 2.9988 | LR: 4.96e-04 | 7003.9ms/step | 8.6 samples/s
|
| 561 |
+
Step 3480/50462 | Loss: 3.0340 | LR: 4.96e-04 | 6999.6ms/step | 8.6 samples/s
|
| 562 |
+
Step 3490/50462 | Loss: 3.0024 | LR: 4.96e-04 | 7003.7ms/step | 8.6 samples/s
|
| 563 |
+
Step 3500/50462 | Loss: 3.0125 | LR: 4.96e-04 | 7004.1ms/step | 8.6 samples/s
|
| 564 |
+
Validation at step 3500: Loss = 2.9992, Perplexity = 20.07
|
| 565 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3500
|
| 566 |
+
Step 3510/50462 | Loss: 3.0054 | LR: 4.96e-04 | 10800.0ms/step | 5.6 samples/s
|
| 567 |
+
Step 3520/50462 | Loss: 3.0265 | LR: 4.96e-04 | 6996.9ms/step | 8.6 samples/s
|
| 568 |
+
Step 3530/50462 | Loss: 2.9760 | LR: 4.96e-04 | 6996.8ms/step | 8.6 samples/s
|
| 569 |
+
Step 3540/50462 | Loss: 3.0349 | LR: 4.96e-04 | 6994.7ms/step | 8.6 samples/s
|
| 570 |
+
Step 3550/50462 | Loss: 3.0334 | LR: 4.96e-04 | 6995.0ms/step | 8.6 samples/s
|
| 571 |
+
Step 3560/50462 | Loss: 3.0091 | LR: 4.96e-04 | 6994.6ms/step | 8.6 samples/s
|
| 572 |
+
Step 3570/50462 | Loss: 3.0159 | LR: 4.96e-04 | 6995.8ms/step | 8.6 samples/s
|
| 573 |
+
Step 3580/50462 | Loss: 2.9684 | LR: 4.96e-04 | 6997.7ms/step | 8.6 samples/s
|
| 574 |
+
Step 3590/50462 | Loss: 2.9992 | LR: 4.96e-04 | 6993.4ms/step | 8.6 samples/s
|
| 575 |
+
Step 3600/50462 | Loss: 2.9951 | LR: 4.96e-04 | 6996.2ms/step | 8.6 samples/s
|
| 576 |
+
Step 3610/50462 | Loss: 3.0092 | LR: 4.96e-04 | 6998.2ms/step | 8.6 samples/s
|
| 577 |
+
Step 3620/50462 | Loss: 2.9983 | LR: 4.96e-04 | 6993.0ms/step | 8.6 samples/s
|
| 578 |
+
Step 3630/50462 | Loss: 3.0109 | LR: 4.96e-04 | 6995.3ms/step | 8.6 samples/s
|
| 579 |
+
Step 3640/50462 | Loss: 3.0164 | LR: 4.96e-04 | 6995.2ms/step | 8.6 samples/s
|
| 580 |
+
Step 3650/50462 | Loss: 2.9972 | LR: 4.96e-04 | 6996.5ms/step | 8.6 samples/s
|
| 581 |
+
Step 3660/50462 | Loss: 2.9938 | LR: 4.96e-04 | 6996.9ms/step | 8.6 samples/s
|
| 582 |
+
Step 3670/50462 | Loss: 3.0070 | LR: 4.96e-04 | 6995.0ms/step | 8.6 samples/s
|
| 583 |
+
Step 3680/50462 | Loss: 2.9592 | LR: 4.96e-04 | 6998.3ms/step | 8.6 samples/s
|
| 584 |
+
Step 3690/50462 | Loss: 2.9704 | LR: 4.96e-04 | 6994.9ms/step | 8.6 samples/s
|
| 585 |
+
Step 3700/50462 | Loss: 2.9944 | LR: 4.96e-04 | 6992.0ms/step | 8.6 samples/s
|
| 586 |
+
Step 3710/50462 | Loss: 2.9934 | LR: 4.96e-04 | 6995.7ms/step | 8.6 samples/s
|
| 587 |
+
Step 3720/50462 | Loss: 2.9773 | LR: 4.96e-04 | 6999.1ms/step | 8.6 samples/s
|
| 588 |
+
Step 3730/50462 | Loss: 2.9717 | LR: 4.96e-04 | 6990.4ms/step | 8.6 samples/s
|
| 589 |
+
Step 3740/50462 | Loss: 2.9750 | LR: 4.96e-04 | 6995.3ms/step | 8.6 samples/s
|
| 590 |
+
Step 3750/50462 | Loss: 2.9603 | LR: 4.96e-04 | 6990.2ms/step | 8.6 samples/s
|
| 591 |
+
Validation at step 3750: Loss = 2.9805, Perplexity = 19.70
|
| 592 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-3750
|
| 593 |
+
Step 3760/50462 | Loss: 2.9941 | LR: 4.95e-04 | 10820.4ms/step | 5.5 samples/s
|
| 594 |
+
Step 3770/50462 | Loss: 2.9756 | LR: 4.95e-04 | 6985.8ms/step | 8.6 samples/s
|
| 595 |
+
Step 3780/50462 | Loss: 3.0174 | LR: 4.95e-04 | 6988.8ms/step | 8.6 samples/s
|
| 596 |
+
Step 3790/50462 | Loss: 3.0043 | LR: 4.95e-04 | 6985.6ms/step | 8.6 samples/s
|
| 597 |
+
Step 3800/50462 | Loss: 2.9699 | LR: 4.95e-04 | 6985.1ms/step | 8.6 samples/s
|
| 598 |
+
Step 3810/50462 | Loss: 2.9569 | LR: 4.95e-04 | 6990.3ms/step | 8.6 samples/s
|
| 599 |
+
Step 3820/50462 | Loss: 2.9567 | LR: 4.95e-04 | 6987.1ms/step | 8.6 samples/s
|
| 600 |
+
Step 3830/50462 | Loss: 3.0193 | LR: 4.95e-04 | 6989.8ms/step | 8.6 samples/s
|
| 601 |
+
Step 3840/50462 | Loss: 2.9586 | LR: 4.95e-04 | 6985.7ms/step | 8.6 samples/s
|
| 602 |
+
Step 3850/50462 | Loss: 3.0179 | LR: 4.95e-04 | 6983.8ms/step | 8.6 samples/s
|
| 603 |
+
Step 3860/50462 | Loss: 2.9404 | LR: 4.95e-04 | 6985.8ms/step | 8.6 samples/s
|
| 604 |
+
Step 3870/50462 | Loss: 2.9509 | LR: 4.95e-04 | 6988.0ms/step | 8.6 samples/s
|
| 605 |
+
Step 3880/50462 | Loss: 2.9673 | LR: 4.95e-04 | 6990.4ms/step | 8.6 samples/s
|
| 606 |
+
Step 3890/50462 | Loss: 2.9607 | LR: 4.95e-04 | 6985.9ms/step | 8.6 samples/s
|
| 607 |
+
Step 3900/50462 | Loss: 3.0096 | LR: 4.95e-04 | 6987.0ms/step | 8.6 samples/s
|
| 608 |
+
Step 3910/50462 | Loss: 2.9598 | LR: 4.95e-04 | 6984.0ms/step | 8.6 samples/s
|
| 609 |
+
Step 3920/50462 | Loss: 2.9733 | LR: 4.95e-04 | 6986.5ms/step | 8.6 samples/s
|
| 610 |
+
Step 3930/50462 | Loss: 2.9840 | LR: 4.95e-04 | 6988.0ms/step | 8.6 samples/s
|
| 611 |
+
Step 3940/50462 | Loss: 2.9849 | LR: 4.95e-04 | 6991.4ms/step | 8.6 samples/s
|
| 612 |
+
Step 3950/50462 | Loss: 2.9811 | LR: 4.95e-04 | 6990.0ms/step | 8.6 samples/s
|
| 613 |
+
Step 3960/50462 | Loss: 2.9710 | LR: 4.95e-04 | 6990.3ms/step | 8.6 samples/s
|
| 614 |
+
Step 3970/50462 | Loss: 3.0009 | LR: 4.95e-04 | 6989.9ms/step | 8.6 samples/s
|
| 615 |
+
Step 3980/50462 | Loss: 2.9627 | LR: 4.95e-04 | 6993.3ms/step | 8.6 samples/s
|
| 616 |
+
Step 3990/50462 | Loss: 2.9685 | LR: 4.95e-04 | 6991.1ms/step | 8.6 samples/s
|
| 617 |
+
Step 4000/50462 | Loss: 2.9596 | LR: 4.95e-04 | 6990.9ms/step | 8.6 samples/s
|
| 618 |
+
Validation at step 4000: Loss = 2.9601, Perplexity = 19.30
|
| 619 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4000
|
| 620 |
+
Step 4010/50462 | Loss: 2.9981 | LR: 4.95e-04 | 10780.6ms/step | 5.6 samples/s
|
| 621 |
+
Step 4020/50462 | Loss: 2.9723 | LR: 4.95e-04 | 6995.1ms/step | 8.6 samples/s
|
| 622 |
+
Step 4030/50462 | Loss: 2.9815 | LR: 4.95e-04 | 6993.8ms/step | 8.6 samples/s
|
| 623 |
+
Step 4040/50462 | Loss: 2.9473 | LR: 4.95e-04 | 6996.5ms/step | 8.6 samples/s
|
| 624 |
+
Step 4050/50462 | Loss: 2.9688 | LR: 4.95e-04 | 6997.4ms/step | 8.6 samples/s
|
| 625 |
+
Step 4060/50462 | Loss: 2.9749 | LR: 4.95e-04 | 6997.4ms/step | 8.6 samples/s
|
| 626 |
+
Step 4070/50462 | Loss: 2.9614 | LR: 4.95e-04 | 7004.9ms/step | 8.6 samples/s
|
| 627 |
+
Step 4080/50462 | Loss: 2.9888 | LR: 4.94e-04 | 6995.8ms/step | 8.6 samples/s
|
| 628 |
+
Step 4090/50462 | Loss: 2.9840 | LR: 4.94e-04 | 6994.3ms/step | 8.6 samples/s
|
| 629 |
+
Step 4100/50462 | Loss: 2.9587 | LR: 4.94e-04 | 6995.2ms/step | 8.6 samples/s
|
| 630 |
+
Step 4110/50462 | Loss: 2.9552 | LR: 4.94e-04 | 6994.8ms/step | 8.6 samples/s
|
| 631 |
+
Step 4120/50462 | Loss: 2.9543 | LR: 4.94e-04 | 6997.1ms/step | 8.6 samples/s
|
| 632 |
+
Step 4130/50462 | Loss: 2.9524 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
|
| 633 |
+
Step 4140/50462 | Loss: 2.9478 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
|
| 634 |
+
Step 4150/50462 | Loss: 2.9574 | LR: 4.94e-04 | 6996.8ms/step | 8.6 samples/s
|
| 635 |
+
Step 4160/50462 | Loss: 2.9604 | LR: 4.94e-04 | 6996.1ms/step | 8.6 samples/s
|
| 636 |
+
Step 4170/50462 | Loss: 2.9167 | LR: 4.94e-04 | 6997.5ms/step | 8.6 samples/s
|
| 637 |
+
Step 4180/50462 | Loss: 2.9211 | LR: 4.94e-04 | 6996.1ms/step | 8.6 samples/s
|
| 638 |
+
Step 4190/50462 | Loss: 2.9405 | LR: 4.94e-04 | 6998.5ms/step | 8.6 samples/s
|
| 639 |
+
Step 4200/50462 | Loss: 2.9799 | LR: 4.94e-04 | 6997.3ms/step | 8.6 samples/s
|
| 640 |
+
Step 4210/50462 | Loss: 2.9519 | LR: 4.94e-04 | 6995.0ms/step | 8.6 samples/s
|
| 641 |
+
Step 4220/50462 | Loss: 2.9385 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
|
| 642 |
+
Step 4230/50462 | Loss: 2.9527 | LR: 4.94e-04 | 6997.7ms/step | 8.6 samples/s
|
| 643 |
+
Step 4240/50462 | Loss: 2.9633 | LR: 4.94e-04 | 6997.3ms/step | 8.6 samples/s
|
| 644 |
+
Step 4250/50462 | Loss: 2.9604 | LR: 4.94e-04 | 6996.6ms/step | 8.6 samples/s
|
| 645 |
+
Validation at step 4250: Loss = 2.9433, Perplexity = 18.98
|
| 646 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4250
|
| 647 |
+
Step 4260/50462 | Loss: 2.9435 | LR: 4.94e-04 | 10779.6ms/step | 5.6 samples/s
|
| 648 |
+
Step 4270/50462 | Loss: 2.9536 | LR: 4.94e-04 | 6984.5ms/step | 8.6 samples/s
|
| 649 |
+
Step 4280/50462 | Loss: 2.9440 | LR: 4.94e-04 | 6987.6ms/step | 8.6 samples/s
|
| 650 |
+
Step 4290/50462 | Loss: 2.9702 | LR: 4.94e-04 | 6994.7ms/step | 8.6 samples/s
|
| 651 |
+
Step 4300/50462 | Loss: 2.9500 | LR: 4.94e-04 | 6992.7ms/step | 8.6 samples/s
|
| 652 |
+
Step 4310/50462 | Loss: 2.9406 | LR: 4.94e-04 | 6990.2ms/step | 8.6 samples/s
|
| 653 |
+
Step 4320/50462 | Loss: 2.9522 | LR: 4.94e-04 | 6991.9ms/step | 8.6 samples/s
|
| 654 |
+
Step 4330/50462 | Loss: 2.9094 | LR: 4.94e-04 | 6992.1ms/step | 8.6 samples/s
|
| 655 |
+
Step 4340/50462 | Loss: 2.9586 | LR: 4.94e-04 | 6989.2ms/step | 8.6 samples/s
|
| 656 |
+
Step 4350/50462 | Loss: 2.9547 | LR: 4.94e-04 | 6992.2ms/step | 8.6 samples/s
|
| 657 |
+
Step 4360/50462 | Loss: 2.9547 | LR: 4.94e-04 | 6997.4ms/step | 8.6 samples/s
|
| 658 |
+
Step 4370/50462 | Loss: 2.9486 | LR: 4.93e-04 | 7023.0ms/step | 8.5 samples/s
|
| 659 |
+
Step 4380/50462 | Loss: 2.9227 | LR: 4.93e-04 | 7033.4ms/step | 8.5 samples/s
|
| 660 |
+
Step 4390/50462 | Loss: 2.9447 | LR: 4.93e-04 | 7029.2ms/step | 8.5 samples/s
|
| 661 |
+
Step 4400/50462 | Loss: 2.9726 | LR: 4.93e-04 | 7028.9ms/step | 8.5 samples/s
|
| 662 |
+
Step 4410/50462 | Loss: 2.9471 | LR: 4.93e-04 | 7015.8ms/step | 8.6 samples/s
|
| 663 |
+
Step 4420/50462 | Loss: 2.9220 | LR: 4.93e-04 | 7021.1ms/step | 8.5 samples/s
|
| 664 |
+
Step 4430/50462 | Loss: 2.9621 | LR: 4.93e-04 | 7015.4ms/step | 8.6 samples/s
|
| 665 |
+
Step 4440/50462 | Loss: 2.8961 | LR: 4.93e-04 | 6995.7ms/step | 8.6 samples/s
|
| 666 |
+
Step 4450/50462 | Loss: 2.9816 | LR: 4.93e-04 | 6988.4ms/step | 8.6 samples/s
|
| 667 |
+
Step 4460/50462 | Loss: 2.9553 | LR: 4.93e-04 | 6989.8ms/step | 8.6 samples/s
|
| 668 |
+
Step 4470/50462 | Loss: 2.9377 | LR: 4.93e-04 | 6990.9ms/step | 8.6 samples/s
|
| 669 |
+
Step 4480/50462 | Loss: 2.9308 | LR: 4.93e-04 | 7017.4ms/step | 8.6 samples/s
|
| 670 |
+
Step 4490/50462 | Loss: 2.9559 | LR: 4.93e-04 | 7020.0ms/step | 8.5 samples/s
|
| 671 |
+
Step 4500/50462 | Loss: 2.9532 | LR: 4.93e-04 | 7015.9ms/step | 8.6 samples/s
|
| 672 |
+
Validation at step 4500: Loss = 2.9257, Perplexity = 18.65
|
| 673 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4500
|
| 674 |
+
Step 4510/50462 | Loss: 2.9406 | LR: 4.93e-04 | 10835.1ms/step | 5.5 samples/s
|
| 675 |
+
Step 4520/50462 | Loss: 2.9427 | LR: 4.93e-04 | 6984.9ms/step | 8.6 samples/s
|
| 676 |
+
Step 4530/50462 | Loss: 2.9319 | LR: 4.93e-04 | 6986.4ms/step | 8.6 samples/s
|
| 677 |
+
Step 4540/50462 | Loss: 2.9591 | LR: 4.93e-04 | 6988.8ms/step | 8.6 samples/s
|
| 678 |
+
Step 4550/50462 | Loss: 2.8968 | LR: 4.93e-04 | 6994.0ms/step | 8.6 samples/s
|
| 679 |
+
Step 4560/50462 | Loss: 2.9353 | LR: 4.93e-04 | 6996.5ms/step | 8.6 samples/s
|
| 680 |
+
Step 4570/50462 | Loss: 2.9200 | LR: 4.93e-04 | 6998.4ms/step | 8.6 samples/s
|
| 681 |
+
Step 4580/50462 | Loss: 2.9691 | LR: 4.93e-04 | 7001.0ms/step | 8.6 samples/s
|
| 682 |
+
Step 4590/50462 | Loss: 2.9395 | LR: 4.93e-04 | 7001.3ms/step | 8.6 samples/s
|
| 683 |
+
Step 4600/50462 | Loss: 2.9255 | LR: 4.93e-04 | 7000.8ms/step | 8.6 samples/s
|
| 684 |
+
Step 4610/50462 | Loss: 2.9342 | LR: 4.93e-04 | 6999.3ms/step | 8.6 samples/s
|
| 685 |
+
Step 4620/50462 | Loss: 2.9498 | LR: 4.93e-04 | 7002.2ms/step | 8.6 samples/s
|
| 686 |
+
Step 4630/50462 | Loss: 2.9489 | LR: 4.93e-04 | 7002.7ms/step | 8.6 samples/s
|
| 687 |
+
Step 4640/50462 | Loss: 2.8864 | LR: 4.92e-04 | 7004.4ms/step | 8.6 samples/s
|
| 688 |
+
Step 4650/50462 | Loss: 2.9097 | LR: 4.92e-04 | 7003.4ms/step | 8.6 samples/s
|
| 689 |
+
Step 4660/50462 | Loss: 2.9420 | LR: 4.92e-04 | 7003.1ms/step | 8.6 samples/s
|
| 690 |
+
Step 4670/50462 | Loss: 2.9389 | LR: 4.92e-04 | 7002.2ms/step | 8.6 samples/s
|
| 691 |
+
Step 4680/50462 | Loss: 2.9490 | LR: 4.92e-04 | 6998.3ms/step | 8.6 samples/s
|
| 692 |
+
Step 4690/50462 | Loss: 2.9286 | LR: 4.92e-04 | 6999.6ms/step | 8.6 samples/s
|
| 693 |
+
Step 4700/50462 | Loss: 2.9467 | LR: 4.92e-04 | 6999.5ms/step | 8.6 samples/s
|
| 694 |
+
Step 4710/50462 | Loss: 2.9263 | LR: 4.92e-04 | 6996.3ms/step | 8.6 samples/s
|
| 695 |
+
Step 4720/50462 | Loss: 2.9237 | LR: 4.92e-04 | 6999.5ms/step | 8.6 samples/s
|
| 696 |
+
Step 4730/50462 | Loss: 2.9466 | LR: 4.92e-04 | 7001.0ms/step | 8.6 samples/s
|
| 697 |
+
Step 4740/50462 | Loss: 2.9121 | LR: 4.92e-04 | 6999.4ms/step | 8.6 samples/s
|
| 698 |
+
Step 4750/50462 | Loss: 2.9019 | LR: 4.92e-04 | 6992.7ms/step | 8.6 samples/s
|
| 699 |
+
Validation at step 4750: Loss = 2.9103, Perplexity = 18.36
|
| 700 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750
|
| 701 |
+
Step 4760/50462 | Loss: 2.9249 | LR: 4.92e-04 | 10842.3ms/step | 5.5 samples/s
|
| 702 |
+
Step 4770/50462 | Loss: 2.9126 | LR: 4.92e-04 | 7000.6ms/step | 8.6 samples/s
|
| 703 |
+
Step 4780/50462 | Loss: 2.9187 | LR: 4.92e-04 | 7000.4ms/step | 8.6 samples/s
|
| 704 |
+
Step 4790/50462 | Loss: 2.9223 | LR: 4.92e-04 | 6999.1ms/step | 8.6 samples/s
|
| 705 |
+
Step 4800/50462 | Loss: 2.9357 | LR: 4.92e-04 | 7009.3ms/step | 8.6 samples/s
|
| 706 |
+
Step 4810/50462 | Loss: 2.9072 | LR: 4.92e-04 | 6997.4ms/step | 8.6 samples/s
|
| 707 |
+
Step 4820/50462 | Loss: 2.9050 | LR: 4.92e-04 | 6997.2ms/step | 8.6 samples/s
|
| 708 |
+
Step 4830/50462 | Loss: 2.9059 | LR: 4.92e-04 | 6995.3ms/step | 8.6 samples/s
|
| 709 |
+
Step 4840/50462 | Loss: 2.9244 | LR: 4.92e-04 | 6998.3ms/step | 8.6 samples/s
|
| 710 |
+
Step 4850/50462 | Loss: 2.9042 | LR: 4.92e-04 | 6996.2ms/step | 8.6 samples/s
|
| 711 |
+
Step 4860/50462 | Loss: 2.9137 | LR: 4.92e-04 | 6993.9ms/step | 8.6 samples/s
|
| 712 |
+
W0823 14:19:43.353000 5032 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
|
| 713 |
+
W0823 14:19:43.357000 5032 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 19202 closing signal SIGTERM
|
| 714 |
+
W0823 14:19:43.360000 5032 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 19203 closing signal SIGTERM
|
| 715 |
+
W0823 14:19:43.361000 5032 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 19205 closing signal SIGTERM
|
| 716 |
+
Traceback (most recent call last):
|
| 717 |
+
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 718 |
+
File "<frozen runpy>", line 88, in _run_code
|
| 719 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 994, in <module>
|
| 720 |
+
main()
|
| 721 |
+
~~~~^^
|
| 722 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
|
| 723 |
+
return f(*args, **kwargs)
|
| 724 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
|
| 725 |
+
run(args)
|
| 726 |
+
~~~^^^^^^
|
| 727 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
|
| 728 |
+
elastic_launch(
|
| 729 |
+
~~~~~~~~~~~~~~~
|
| 730 |
+
config=config,
|
| 731 |
+
~~~~~~~~~~~~~~
|
| 732 |
+
entrypoint=cmd,
|
| 733 |
+
~~~~~~~~~~~~~~~
|
| 734 |
+
)(*cmd_args)
|
| 735 |
+
~^^^^^^^^^^^
|
| 736 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
|
| 737 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 738 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
|
| 739 |
+
result = agent.run()
|
| 740 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
|
| 741 |
+
result = f(*args, **kwargs)
|
| 742 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
|
| 743 |
+
result = self._invoke_run(role)
|
| 744 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
|
| 745 |
+
time.sleep(monitor_interval)
|
| 746 |
+
~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
| 747 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
|
| 748 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 749 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 5032 got signal: 15
|
epfl_final/train_logs/p2_1p2b_dense_seed70_4gpu_h1004.log
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_040030
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_040030",
|
| 16 |
+
"resume_from_checkpoint": null,
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 1000,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Initializing model from scratch (random weights)...
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
================================================================================
|
| 77 |
+
MODEL ARCHITECTURE
|
| 78 |
+
================================================================================
|
| 79 |
+
Model config:
|
| 80 |
+
vocab_size: 151936
|
| 81 |
+
hidden_size: 1536
|
| 82 |
+
num_hidden_layers: 28
|
| 83 |
+
num_attention_heads: 12
|
| 84 |
+
num_key_value_heads: 4
|
| 85 |
+
head_dim: 128
|
| 86 |
+
intermediate_size: 6144
|
| 87 |
+
max_position_embeddings: 32768
|
| 88 |
+
rope_theta: 1000000
|
| 89 |
+
rope_train_position_scale_set: None
|
| 90 |
+
attention_type: sparse_stochastic
|
| 91 |
+
sparse_block_size: 128
|
| 92 |
+
sparse_num_random_blocks: 31
|
| 93 |
+
sparse_mask_pool_size: 1
|
| 94 |
+
sparse_max_seq_length: 8192
|
| 95 |
+
|
| 96 |
+
Parameter count:
|
| 97 |
+
Total parameters: 1,202,352,640
|
| 98 |
+
Trainable parameters: 1,202,352,640
|
| 99 |
+
Non-trainable parameters: 0
|
| 100 |
+
Model size (BF16): 2.40 GB
|
| 101 |
+
|
| 102 |
+
--------------------------------------------------------------------------------
|
| 103 |
+
LAYER-BY-LAYER SUMMARY
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
================================================================================================================================================================
|
| 106 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 109 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 110 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 111 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 112 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 141 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 142 |
+
================================================================================================================================================================
|
| 143 |
+
Total params: 1,435,726,336
|
| 144 |
+
Trainable params: 1,435,726,336
|
| 145 |
+
Non-trainable params: 0
|
| 146 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 147 |
+
================================================================================================================================================================
|
| 148 |
+
Input size (MB): 0.07
|
| 149 |
+
Forward/backward pass size (MB): 26218.59
|
| 150 |
+
Params size (MB): 2871.45
|
| 151 |
+
Estimated Total Size (MB): 29090.11
|
| 152 |
+
================================================================================================================================================================
|
| 153 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 154 |
+
================================================================================
|
| 155 |
+
|
| 156 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 157 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 163 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 164 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 165 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 166 |
+
Dataset split: 3027764 train, 159356 validation
|
| 167 |
+
Training examples (packed, train split): 3027764
|
| 168 |
+
Validation examples (packed, val split): 159356
|
| 169 |
+
Per-device batch size: 1
|
| 170 |
+
World size (GPUs): 4
|
| 171 |
+
Gradient accumulation steps: 15
|
| 172 |
+
Effective batch size (seqs/step): 60
|
| 173 |
+
Sequence length: 8192
|
| 174 |
+
Tokens per optimizer step: 491,520
|
| 175 |
+
Micro-batches per epoch (per rank): 756941
|
| 176 |
+
Optimizer steps per epoch: 50462
|
| 177 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 178 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 179 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 180 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 181 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_040101-y8clx6eu
|
| 182 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 183 |
+
wandb: Syncing run 1.2b dense 8k seed70
|
| 184 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 185 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/y8clx6eu
|
| 186 |
+
Starting training...
|
| 187 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 188 |
+
W0823 04:01:36.807000 4252 site-packages/torch/distributed/elastic/agent/server/api.py:739] Received 15 death signal, shutting down workers
|
| 189 |
+
W0823 04:01:36.815000 4252 site-packages/torch/distributed/elastic/multiprocessing/api.py:1012] Sending process 4321 closing signal SIGTERM
|
| 190 |
+
Traceback (most recent call last):
|
| 191 |
+
File "<frozen runpy>", line 198, in _run_module_as_main
|
| 192 |
+
File "<frozen runpy>", line 88, in _run_code
|
| 193 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 994, in <module>
|
| 194 |
+
main()
|
| 195 |
+
~~~~^^
|
| 196 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 362, in wrapper
|
| 197 |
+
return f(*args, **kwargs)
|
| 198 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 990, in main
|
| 199 |
+
run(args)
|
| 200 |
+
~~~^^^^^^
|
| 201 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/run.py", line 981, in run
|
| 202 |
+
elastic_launch(
|
| 203 |
+
~~~~~~~~~~~~~~~
|
| 204 |
+
config=config,
|
| 205 |
+
~~~~~~~~~~~~~~
|
| 206 |
+
entrypoint=cmd,
|
| 207 |
+
~~~~~~~~~~~~~~~
|
| 208 |
+
)(*cmd_args)
|
| 209 |
+
~^^^^^^^^^^^
|
| 210 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 170, in __call__
|
| 211 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 212 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/launcher/api.py", line 308, in launch_agent
|
| 213 |
+
result = agent.run()
|
| 214 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/metrics/api.py", line 134, in wrapper
|
| 215 |
+
result = f(*args, **kwargs)
|
| 216 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 731, in run
|
| 217 |
+
result = self._invoke_run(role)
|
| 218 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/agent/server/api.py", line 908, in _invoke_run
|
| 219 |
+
time.sleep(monitor_interval)
|
| 220 |
+
~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
| 221 |
+
File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 86, in _terminate_process_handler
|
| 222 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 223 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 4252 got signal: 15
|
epfl_final/train_logs/p2_1p2b_dense_seed70_5gpu.log
ADDED
|
@@ -0,0 +1,806 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 12,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 5gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750
|
| 73 |
+
Checkpoint activations: False
|
| 74 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
================================================================================
|
| 78 |
+
MODEL ARCHITECTURE
|
| 79 |
+
================================================================================
|
| 80 |
+
Model config:
|
| 81 |
+
vocab_size: 151936
|
| 82 |
+
hidden_size: 1536
|
| 83 |
+
num_hidden_layers: 28
|
| 84 |
+
num_attention_heads: 12
|
| 85 |
+
num_key_value_heads: 4
|
| 86 |
+
head_dim: 128
|
| 87 |
+
intermediate_size: 6144
|
| 88 |
+
max_position_embeddings: 32768
|
| 89 |
+
rope_theta: 1000000
|
| 90 |
+
rope_train_position_scale_set: None
|
| 91 |
+
attention_type: sparse_stochastic
|
| 92 |
+
sparse_block_size: 128
|
| 93 |
+
sparse_num_random_blocks: 31
|
| 94 |
+
sparse_mask_pool_size: 1
|
| 95 |
+
sparse_max_seq_length: 8192
|
| 96 |
+
|
| 97 |
+
Parameter count:
|
| 98 |
+
Total parameters: 1,202,352,640
|
| 99 |
+
Trainable parameters: 1,202,352,640
|
| 100 |
+
Non-trainable parameters: 0
|
| 101 |
+
Model size (BF16): 2.40 GB
|
| 102 |
+
|
| 103 |
+
--------------------------------------------------------------------------------
|
| 104 |
+
LAYER-BY-LAYER SUMMARY
|
| 105 |
+
--------------------------------------------------------------------------------
|
| 106 |
+
================================================================================================================================================================
|
| 107 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 108 |
+
================================================================================================================================================================
|
| 109 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 110 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 111 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 112 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 142 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
Total params: 1,435,726,336
|
| 145 |
+
Trainable params: 1,435,726,336
|
| 146 |
+
Non-trainable params: 0
|
| 147 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 148 |
+
================================================================================================================================================================
|
| 149 |
+
Input size (MB): 0.07
|
| 150 |
+
Forward/backward pass size (MB): 26218.59
|
| 151 |
+
Params size (MB): 2871.45
|
| 152 |
+
Estimated Total Size (MB): 29090.11
|
| 153 |
+
================================================================================================================================================================
|
| 154 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 155 |
+
================================================================================
|
| 156 |
+
|
| 157 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 162 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 163 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 164 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 165 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 166 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 167 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 168 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 169 |
+
Dataset split: 3027764 train, 159356 validation
|
| 170 |
+
Training examples (packed, train split): 3027764
|
| 171 |
+
Validation examples (packed, val split): 159356
|
| 172 |
+
Per-device batch size: 1
|
| 173 |
+
World size (GPUs): 5
|
| 174 |
+
Gradient accumulation steps: 12
|
| 175 |
+
Effective batch size (seqs/step): 60
|
| 176 |
+
Sequence length: 8192
|
| 177 |
+
Tokens per optimizer step: 491,520
|
| 178 |
+
Micro-batches per epoch (per rank): 605553
|
| 179 |
+
Optimizer steps per epoch: 50462
|
| 180 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 181 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-4750...
|
| 182 |
+
Preserving checkpoint LR state for exact resume: current=[0.00049205500848748], schedule_base=[0.0005]
|
| 183 |
+
Resumed from checkpoint at step 4750 (epoch 0, micro_step 71250)
|
| 184 |
+
Resuming training from step 4750
|
| 185 |
+
Resuming wandb run rgmkfpnl
|
| 186 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 187 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 188 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 189 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_142101-rgmkfpnl
|
| 190 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 191 |
+
wandb: Resuming run 1.2b dense 8k seed70 5gpu resume
|
| 192 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 193 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 194 |
+
Starting training...
|
| 195 |
+
Validation at step 4750: Loss = 2.9765, Perplexity = 19.62
|
| 196 |
+
wandb: WARNING Tried to log to step 4750 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 197 |
+
Switching attention mode for step 4751: dense (dense step interval)
|
| 198 |
+
Step 4760/50462 | Loss: 2.7898 | LR: 4.92e-04 | 25557.5ms/step | 2.3 samples/s
|
| 199 |
+
wandb: WARNING Tried to log to step 4760 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Step 4770/50462 | Loss: 2.9189 | LR: 4.92e-04 | 5606.2ms/step | 10.7 samples/s
|
| 201 |
+
wandb: WARNING Tried to log to step 4770 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 202 |
+
Step 4780/50462 | Loss: 2.8885 | LR: 4.92e-04 | 5613.3ms/step | 10.7 samples/s
|
| 203 |
+
wandb: WARNING Tried to log to step 4780 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 204 |
+
Step 4790/50462 | Loss: 2.9275 | LR: 4.92e-04 | 5611.0ms/step | 10.7 samples/s
|
| 205 |
+
wandb: WARNING Tried to log to step 4790 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 206 |
+
Step 4800/50462 | Loss: 2.9016 | LR: 4.92e-04 | 5616.0ms/step | 10.7 samples/s
|
| 207 |
+
wandb: WARNING Tried to log to step 4800 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 208 |
+
Step 4810/50462 | Loss: 2.9204 | LR: 4.92e-04 | 5617.1ms/step | 10.7 samples/s
|
| 209 |
+
wandb: WARNING Tried to log to step 4810 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Step 4820/50462 | Loss: 2.9325 | LR: 4.92e-04 | 5620.1ms/step | 10.7 samples/s
|
| 211 |
+
wandb: WARNING Tried to log to step 4820 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 212 |
+
Step 4830/50462 | Loss: 2.8947 | LR: 4.92e-04 | 5620.8ms/step | 10.7 samples/s
|
| 213 |
+
wandb: WARNING Tried to log to step 4830 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Step 4840/50462 | Loss: 2.9317 | LR: 4.92e-04 | 5617.2ms/step | 10.7 samples/s
|
| 215 |
+
wandb: WARNING Tried to log to step 4840 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 216 |
+
Step 4850/50462 | Loss: 2.9170 | LR: 4.92e-04 | 5620.6ms/step | 10.7 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 4850 that is less than the current step 4851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 4860/50462 | Loss: 2.8961 | LR: 4.92e-04 | 5621.4ms/step | 10.7 samples/s
|
| 219 |
+
Step 4870/50462 | Loss: 2.9214 | LR: 4.92e-04 | 5624.5ms/step | 10.7 samples/s
|
| 220 |
+
Step 4880/50462 | Loss: 2.9081 | LR: 4.92e-04 | 5627.3ms/step | 10.7 samples/s
|
| 221 |
+
Step 4890/50462 | Loss: 2.8927 | LR: 4.91e-04 | 5624.2ms/step | 10.7 samples/s
|
| 222 |
+
Step 4900/50462 | Loss: 2.9027 | LR: 4.91e-04 | 5619.8ms/step | 10.7 samples/s
|
| 223 |
+
Step 4910/50462 | Loss: 2.9412 | LR: 4.91e-04 | 5622.9ms/step | 10.7 samples/s
|
| 224 |
+
Step 4920/50462 | Loss: 2.9107 | LR: 4.91e-04 | 5620.5ms/step | 10.7 samples/s
|
| 225 |
+
Step 4930/50462 | Loss: 2.8734 | LR: 4.91e-04 | 5614.3ms/step | 10.7 samples/s
|
| 226 |
+
Step 4940/50462 | Loss: 2.9371 | LR: 4.91e-04 | 5594.0ms/step | 10.7 samples/s
|
| 227 |
+
Step 4950/50462 | Loss: 2.9249 | LR: 4.91e-04 | 5587.1ms/step | 10.7 samples/s
|
| 228 |
+
Step 4960/50462 | Loss: 2.9282 | LR: 4.91e-04 | 5588.0ms/step | 10.7 samples/s
|
| 229 |
+
Step 4970/50462 | Loss: 2.9214 | LR: 4.91e-04 | 5588.2ms/step | 10.7 samples/s
|
| 230 |
+
Step 4980/50462 | Loss: 2.9222 | LR: 4.91e-04 | 5600.1ms/step | 10.7 samples/s
|
| 231 |
+
Step 4990/50462 | Loss: 2.9054 | LR: 4.91e-04 | 5609.8ms/step | 10.7 samples/s
|
| 232 |
+
Step 5000/50462 | Loss: 2.9347 | LR: 4.91e-04 | 5621.1ms/step | 10.7 samples/s
|
| 233 |
+
Validation at step 5000: Loss = 2.9632, Perplexity = 19.36
|
| 234 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5000
|
| 235 |
+
Step 5010/50462 | Loss: 2.9167 | LR: 4.91e-04 | 8408.3ms/step | 7.1 samples/s
|
| 236 |
+
Step 5020/50462 | Loss: 2.9162 | LR: 4.91e-04 | 5628.8ms/step | 10.7 samples/s
|
| 237 |
+
Step 5030/50462 | Loss: 2.9067 | LR: 4.91e-04 | 5630.2ms/step | 10.7 samples/s
|
| 238 |
+
Step 5040/50462 | Loss: 2.8993 | LR: 4.91e-04 | 5630.2ms/step | 10.7 samples/s
|
| 239 |
+
Step 5050/50462 | Loss: 2.9362 | LR: 4.91e-04 | 5629.8ms/step | 10.7 samples/s
|
| 240 |
+
Step 5060/50462 | Loss: 2.9193 | LR: 4.91e-04 | 5630.6ms/step | 10.7 samples/s
|
| 241 |
+
Step 5070/50462 | Loss: 2.9416 | LR: 4.91e-04 | 5631.0ms/step | 10.7 samples/s
|
| 242 |
+
Step 5080/50462 | Loss: 2.8938 | LR: 4.91e-04 | 5627.7ms/step | 10.7 samples/s
|
| 243 |
+
Step 5090/50462 | Loss: 2.8894 | LR: 4.91e-04 | 5629.1ms/step | 10.7 samples/s
|
| 244 |
+
Step 5100/50462 | Loss: 2.9151 | LR: 4.91e-04 | 5624.0ms/step | 10.7 samples/s
|
| 245 |
+
Step 5110/50462 | Loss: 2.9174 | LR: 4.91e-04 | 5626.5ms/step | 10.7 samples/s
|
| 246 |
+
Step 5120/50462 | Loss: 2.8755 | LR: 4.91e-04 | 5629.6ms/step | 10.7 samples/s
|
| 247 |
+
Step 5130/50462 | Loss: 2.9104 | LR: 4.90e-04 | 5624.6ms/step | 10.7 samples/s
|
| 248 |
+
Step 5140/50462 | Loss: 2.8762 | LR: 4.90e-04 | 5625.8ms/step | 10.7 samples/s
|
| 249 |
+
Step 5150/50462 | Loss: 2.8867 | LR: 4.90e-04 | 5629.5ms/step | 10.7 samples/s
|
| 250 |
+
Step 5160/50462 | Loss: 2.8901 | LR: 4.90e-04 | 5631.5ms/step | 10.7 samples/s
|
| 251 |
+
Step 5170/50462 | Loss: 2.9315 | LR: 4.90e-04 | 5628.2ms/step | 10.7 samples/s
|
| 252 |
+
Step 5180/50462 | Loss: 2.9286 | LR: 4.90e-04 | 5628.2ms/step | 10.7 samples/s
|
| 253 |
+
Step 5190/50462 | Loss: 2.8828 | LR: 4.90e-04 | 5626.5ms/step | 10.7 samples/s
|
| 254 |
+
Step 5200/50462 | Loss: 2.9138 | LR: 4.90e-04 | 5628.6ms/step | 10.7 samples/s
|
| 255 |
+
Step 5210/50462 | Loss: 2.8465 | LR: 4.90e-04 | 5630.6ms/step | 10.7 samples/s
|
| 256 |
+
Step 5220/50462 | Loss: 2.8790 | LR: 4.90e-04 | 5629.3ms/step | 10.7 samples/s
|
| 257 |
+
Step 5230/50462 | Loss: 2.9057 | LR: 4.90e-04 | 5631.1ms/step | 10.7 samples/s
|
| 258 |
+
Step 5240/50462 | Loss: 2.8852 | LR: 4.90e-04 | 5630.2ms/step | 10.7 samples/s
|
| 259 |
+
Step 5250/50462 | Loss: 2.9079 | LR: 4.90e-04 | 5625.9ms/step | 10.7 samples/s
|
| 260 |
+
Validation at step 5250: Loss = 2.9504, Perplexity = 19.11
|
| 261 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5250
|
| 262 |
+
Step 5260/50462 | Loss: 2.8622 | LR: 4.90e-04 | 8411.9ms/step | 7.1 samples/s
|
| 263 |
+
Step 5270/50462 | Loss: 2.8674 | LR: 4.90e-04 | 5626.8ms/step | 10.7 samples/s
|
| 264 |
+
Step 5280/50462 | Loss: 2.8786 | LR: 4.90e-04 | 5629.7ms/step | 10.7 samples/s
|
| 265 |
+
Step 5290/50462 | Loss: 2.9092 | LR: 4.90e-04 | 5632.1ms/step | 10.7 samples/s
|
| 266 |
+
Step 5300/50462 | Loss: 2.8912 | LR: 4.90e-04 | 5630.1ms/step | 10.7 samples/s
|
| 267 |
+
Step 5310/50462 | Loss: 2.8852 | LR: 4.90e-04 | 5632.0ms/step | 10.7 samples/s
|
| 268 |
+
Step 5320/50462 | Loss: 2.8587 | LR: 4.90e-04 | 5634.3ms/step | 10.6 samples/s
|
| 269 |
+
Step 5330/50462 | Loss: 2.8758 | LR: 4.90e-04 | 5630.3ms/step | 10.7 samples/s
|
| 270 |
+
Step 5340/50462 | Loss: 2.9077 | LR: 4.90e-04 | 5628.4ms/step | 10.7 samples/s
|
| 271 |
+
Step 5350/50462 | Loss: 2.8832 | LR: 4.90e-04 | 5631.1ms/step | 10.7 samples/s
|
| 272 |
+
Step 5360/50462 | Loss: 2.9022 | LR: 4.89e-04 | 5630.8ms/step | 10.7 samples/s
|
| 273 |
+
Step 5370/50462 | Loss: 2.8662 | LR: 4.89e-04 | 5632.8ms/step | 10.7 samples/s
|
| 274 |
+
Step 5380/50462 | Loss: 2.8646 | LR: 4.89e-04 | 5631.1ms/step | 10.7 samples/s
|
| 275 |
+
Step 5390/50462 | Loss: 2.9104 | LR: 4.89e-04 | 5630.2ms/step | 10.7 samples/s
|
| 276 |
+
Step 5400/50462 | Loss: 2.9088 | LR: 4.89e-04 | 5629.6ms/step | 10.7 samples/s
|
| 277 |
+
Step 5410/50462 | Loss: 2.9090 | LR: 4.89e-04 | 5629.7ms/step | 10.7 samples/s
|
| 278 |
+
Step 5420/50462 | Loss: 2.8929 | LR: 4.89e-04 | 5627.7ms/step | 10.7 samples/s
|
| 279 |
+
Step 5430/50462 | Loss: 2.9173 | LR: 4.89e-04 | 5631.1ms/step | 10.7 samples/s
|
| 280 |
+
Step 5440/50462 | Loss: 2.9053 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 281 |
+
Step 5450/50462 | Loss: 2.9171 | LR: 4.89e-04 | 5629.6ms/step | 10.7 samples/s
|
| 282 |
+
Step 5460/50462 | Loss: 2.8715 | LR: 4.89e-04 | 5631.5ms/step | 10.7 samples/s
|
| 283 |
+
Step 5470/50462 | Loss: 2.8659 | LR: 4.89e-04 | 5632.2ms/step | 10.7 samples/s
|
| 284 |
+
Step 5480/50462 | Loss: 2.9038 | LR: 4.89e-04 | 5631.0ms/step | 10.7 samples/s
|
| 285 |
+
Step 5490/50462 | Loss: 2.8241 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 286 |
+
Step 5500/50462 | Loss: 2.8918 | LR: 4.89e-04 | 5631.3ms/step | 10.7 samples/s
|
| 287 |
+
Validation at step 5500: Loss = 2.9392, Perplexity = 18.90
|
| 288 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5500
|
| 289 |
+
Step 5510/50462 | Loss: 2.8884 | LR: 4.89e-04 | 8372.1ms/step | 7.2 samples/s
|
| 290 |
+
Step 5520/50462 | Loss: 2.8531 | LR: 4.89e-04 | 5630.6ms/step | 10.7 samples/s
|
| 291 |
+
Step 5530/50462 | Loss: 2.8670 | LR: 4.89e-04 | 5631.0ms/step | 10.7 samples/s
|
| 292 |
+
Step 5540/50462 | Loss: 2.8558 | LR: 4.89e-04 | 5632.1ms/step | 10.7 samples/s
|
| 293 |
+
Step 5550/50462 | Loss: 2.8815 | LR: 4.89e-04 | 5628.9ms/step | 10.7 samples/s
|
| 294 |
+
Step 5560/50462 | Loss: 2.8765 | LR: 4.89e-04 | 5630.0ms/step | 10.7 samples/s
|
| 295 |
+
Step 5570/50462 | Loss: 2.8838 | LR: 4.88e-04 | 5630.8ms/step | 10.7 samples/s
|
| 296 |
+
Step 5580/50462 | Loss: 2.9248 | LR: 4.88e-04 | 5632.9ms/step | 10.7 samples/s
|
| 297 |
+
Step 5590/50462 | Loss: 2.8654 | LR: 4.88e-04 | 5632.0ms/step | 10.7 samples/s
|
| 298 |
+
Step 5600/50462 | Loss: 2.9087 | LR: 4.88e-04 | 5630.2ms/step | 10.7 samples/s
|
| 299 |
+
Step 5610/50462 | Loss: 2.8404 | LR: 4.88e-04 | 5628.7ms/step | 10.7 samples/s
|
| 300 |
+
Step 5620/50462 | Loss: 2.8812 | LR: 4.88e-04 | 5629.5ms/step | 10.7 samples/s
|
| 301 |
+
Step 5630/50462 | Loss: 2.8908 | LR: 4.88e-04 | 5631.6ms/step | 10.7 samples/s
|
| 302 |
+
Step 5640/50462 | Loss: 2.8975 | LR: 4.88e-04 | 5631.9ms/step | 10.7 samples/s
|
| 303 |
+
Step 5650/50462 | Loss: 2.9143 | LR: 4.88e-04 | 5631.8ms/step | 10.7 samples/s
|
| 304 |
+
Step 5660/50462 | Loss: 2.8830 | LR: 4.88e-04 | 5631.2ms/step | 10.7 samples/s
|
| 305 |
+
Step 5670/50462 | Loss: 2.8973 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 306 |
+
Step 5680/50462 | Loss: 2.8657 | LR: 4.88e-04 | 5632.2ms/step | 10.7 samples/s
|
| 307 |
+
Step 5690/50462 | Loss: 2.8848 | LR: 4.88e-04 | 5632.3ms/step | 10.7 samples/s
|
| 308 |
+
Step 5700/50462 | Loss: 2.8677 | LR: 4.88e-04 | 5631.7ms/step | 10.7 samples/s
|
| 309 |
+
Step 5710/50462 | Loss: 2.8588 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 310 |
+
Step 5720/50462 | Loss: 2.8786 | LR: 4.88e-04 | 5631.9ms/step | 10.7 samples/s
|
| 311 |
+
Step 5730/50462 | Loss: 2.8848 | LR: 4.88e-04 | 5631.5ms/step | 10.7 samples/s
|
| 312 |
+
Step 5740/50462 | Loss: 2.9005 | LR: 4.88e-04 | 5630.7ms/step | 10.7 samples/s
|
| 313 |
+
Step 5750/50462 | Loss: 2.8510 | LR: 4.88e-04 | 5632.8ms/step | 10.7 samples/s
|
| 314 |
+
Validation at step 5750: Loss = 2.9269, Perplexity = 18.67
|
| 315 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-5750
|
| 316 |
+
Step 5760/50462 | Loss: 2.8559 | LR: 4.88e-04 | 8387.7ms/step | 7.2 samples/s
|
| 317 |
+
Step 5770/50462 | Loss: 2.8836 | LR: 4.88e-04 | 5627.9ms/step | 10.7 samples/s
|
| 318 |
+
Step 5780/50462 | Loss: 2.8623 | LR: 4.87e-04 | 5627.9ms/step | 10.7 samples/s
|
| 319 |
+
Step 5790/50462 | Loss: 2.8753 | LR: 4.87e-04 | 5628.3ms/step | 10.7 samples/s
|
| 320 |
+
Step 5800/50462 | Loss: 2.8568 | LR: 4.87e-04 | 5627.9ms/step | 10.7 samples/s
|
| 321 |
+
Step 5810/50462 | Loss: 2.8833 | LR: 4.87e-04 | 5628.6ms/step | 10.7 samples/s
|
| 322 |
+
Step 5820/50462 | Loss: 2.9242 | LR: 4.87e-04 | 5628.5ms/step | 10.7 samples/s
|
| 323 |
+
Step 5830/50462 | Loss: 2.8731 | LR: 4.87e-04 | 5629.8ms/step | 10.7 samples/s
|
| 324 |
+
Step 5840/50462 | Loss: 2.8296 | LR: 4.87e-04 | 5630.7ms/step | 10.7 samples/s
|
| 325 |
+
Step 5850/50462 | Loss: 2.8613 | LR: 4.87e-04 | 5629.5ms/step | 10.7 samples/s
|
| 326 |
+
Step 5860/50462 | Loss: 2.8531 | LR: 4.87e-04 | 5630.3ms/step | 10.7 samples/s
|
| 327 |
+
Step 5870/50462 | Loss: 2.8372 | LR: 4.87e-04 | 5630.5ms/step | 10.7 samples/s
|
| 328 |
+
Step 5880/50462 | Loss: 2.8499 | LR: 4.87e-04 | 5633.4ms/step | 10.7 samples/s
|
| 329 |
+
Step 5890/50462 | Loss: 2.9092 | LR: 4.87e-04 | 5630.0ms/step | 10.7 samples/s
|
| 330 |
+
Step 5900/50462 | Loss: 2.8151 | LR: 4.87e-04 | 5630.1ms/step | 10.7 samples/s
|
| 331 |
+
Step 5910/50462 | Loss: 2.8835 | LR: 4.87e-04 | 5630.8ms/step | 10.7 samples/s
|
| 332 |
+
Step 5920/50462 | Loss: 2.8911 | LR: 4.87e-04 | 5631.8ms/step | 10.7 samples/s
|
| 333 |
+
Step 5930/50462 | Loss: 2.8540 | LR: 4.87e-04 | 5629.2ms/step | 10.7 samples/s
|
| 334 |
+
Step 5940/50462 | Loss: 2.8611 | LR: 4.87e-04 | 5631.2ms/step | 10.7 samples/s
|
| 335 |
+
Step 5950/50462 | Loss: 2.8628 | LR: 4.87e-04 | 5628.4ms/step | 10.7 samples/s
|
| 336 |
+
Step 5960/50462 | Loss: 2.8701 | LR: 4.87e-04 | 5626.0ms/step | 10.7 samples/s
|
| 337 |
+
Step 5970/50462 | Loss: 2.8578 | LR: 4.87e-04 | 5627.0ms/step | 10.7 samples/s
|
| 338 |
+
Step 5980/50462 | Loss: 2.9127 | LR: 4.86e-04 | 5626.9ms/step | 10.7 samples/s
|
| 339 |
+
Step 5990/50462 | Loss: 2.8624 | LR: 4.86e-04 | 5628.2ms/step | 10.7 samples/s
|
| 340 |
+
Step 6000/50462 | Loss: 2.8871 | LR: 4.86e-04 | 5628.3ms/step | 10.7 samples/s
|
| 341 |
+
Validation at step 6000: Loss = 2.9159, Perplexity = 18.47
|
| 342 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6000
|
| 343 |
+
Step 6010/50462 | Loss: 2.8736 | LR: 4.86e-04 | 8467.4ms/step | 7.1 samples/s
|
| 344 |
+
Step 6020/50462 | Loss: 2.8459 | LR: 4.86e-04 | 5625.4ms/step | 10.7 samples/s
|
| 345 |
+
Step 6030/50462 | Loss: 2.8760 | LR: 4.86e-04 | 5626.1ms/step | 10.7 samples/s
|
| 346 |
+
Step 6040/50462 | Loss: 2.8854 | LR: 4.86e-04 | 5625.5ms/step | 10.7 samples/s
|
| 347 |
+
Step 6050/50462 | Loss: 2.8499 | LR: 4.86e-04 | 5627.0ms/step | 10.7 samples/s
|
| 348 |
+
Step 6060/50462 | Loss: 2.8394 | LR: 4.86e-04 | 5626.7ms/step | 10.7 samples/s
|
| 349 |
+
Step 6070/50462 | Loss: 2.8636 | LR: 4.86e-04 | 5627.3ms/step | 10.7 samples/s
|
| 350 |
+
Step 6080/50462 | Loss: 2.8463 | LR: 4.86e-04 | 5626.4ms/step | 10.7 samples/s
|
| 351 |
+
Step 6090/50462 | Loss: 2.8518 | LR: 4.86e-04 | 5626.2ms/step | 10.7 samples/s
|
| 352 |
+
Step 6100/50462 | Loss: 2.8666 | LR: 4.86e-04 | 5625.7ms/step | 10.7 samples/s
|
| 353 |
+
Step 6110/50462 | Loss: 2.8935 | LR: 4.86e-04 | 5626.4ms/step | 10.7 samples/s
|
| 354 |
+
Step 6120/50462 | Loss: 2.8419 | LR: 4.86e-04 | 5625.8ms/step | 10.7 samples/s
|
| 355 |
+
Step 6130/50462 | Loss: 2.8559 | LR: 4.86e-04 | 5625.9ms/step | 10.7 samples/s
|
| 356 |
+
Step 6140/50462 | Loss: 2.8538 | LR: 4.86e-04 | 5627.4ms/step | 10.7 samples/s
|
| 357 |
+
Step 6150/50462 | Loss: 2.8704 | LR: 4.86e-04 | 5625.8ms/step | 10.7 samples/s
|
| 358 |
+
Step 6160/50462 | Loss: 2.8512 | LR: 4.86e-04 | 5624.5ms/step | 10.7 samples/s
|
| 359 |
+
Step 6170/50462 | Loss: 2.8844 | LR: 4.85e-04 | 5625.2ms/step | 10.7 samples/s
|
| 360 |
+
Step 6180/50462 | Loss: 2.8691 | LR: 4.85e-04 | 5626.8ms/step | 10.7 samples/s
|
| 361 |
+
Step 6190/50462 | Loss: 2.8418 | LR: 4.85e-04 | 5628.1ms/step | 10.7 samples/s
|
| 362 |
+
Step 6200/50462 | Loss: 2.8738 | LR: 4.85e-04 | 5629.4ms/step | 10.7 samples/s
|
| 363 |
+
Step 6210/50462 | Loss: 2.8575 | LR: 4.85e-04 | 5628.7ms/step | 10.7 samples/s
|
| 364 |
+
Step 6220/50462 | Loss: 2.8286 | LR: 4.85e-04 | 5629.6ms/step | 10.7 samples/s
|
| 365 |
+
Step 6230/50462 | Loss: 2.8197 | LR: 4.85e-04 | 5627.0ms/step | 10.7 samples/s
|
| 366 |
+
Step 6240/50462 | Loss: 2.8540 | LR: 4.85e-04 | 5626.2ms/step | 10.7 samples/s
|
| 367 |
+
Step 6250/50462 | Loss: 2.8531 | LR: 4.85e-04 | 5625.4ms/step | 10.7 samples/s
|
| 368 |
+
Validation at step 6250: Loss = 2.9014, Perplexity = 18.20
|
| 369 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6250
|
| 370 |
+
Step 6260/50462 | Loss: 2.8687 | LR: 4.85e-04 | 8514.2ms/step | 7.0 samples/s
|
| 371 |
+
Step 6270/50462 | Loss: 2.8768 | LR: 4.85e-04 | 5625.1ms/step | 10.7 samples/s
|
| 372 |
+
Step 6280/50462 | Loss: 2.8565 | LR: 4.85e-04 | 5625.4ms/step | 10.7 samples/s
|
| 373 |
+
Step 6290/50462 | Loss: 2.8830 | LR: 4.85e-04 | 5622.4ms/step | 10.7 samples/s
|
| 374 |
+
Step 6300/50462 | Loss: 2.8696 | LR: 4.85e-04 | 5608.7ms/step | 10.7 samples/s
|
| 375 |
+
Step 6310/50462 | Loss: 2.8627 | LR: 4.85e-04 | 5606.0ms/step | 10.7 samples/s
|
| 376 |
+
Step 6320/50462 | Loss: 2.8211 | LR: 4.85e-04 | 5606.3ms/step | 10.7 samples/s
|
| 377 |
+
Step 6330/50462 | Loss: 2.8506 | LR: 4.85e-04 | 5605.5ms/step | 10.7 samples/s
|
| 378 |
+
Step 6340/50462 | Loss: 2.8594 | LR: 4.85e-04 | 5600.9ms/step | 10.7 samples/s
|
| 379 |
+
Step 6350/50462 | Loss: 2.8198 | LR: 4.85e-04 | 5602.0ms/step | 10.7 samples/s
|
| 380 |
+
Step 6360/50462 | Loss: 2.8180 | LR: 4.84e-04 | 5601.4ms/step | 10.7 samples/s
|
| 381 |
+
Step 6370/50462 | Loss: 2.8644 | LR: 4.84e-04 | 5602.0ms/step | 10.7 samples/s
|
| 382 |
+
Step 6380/50462 | Loss: 2.8690 | LR: 4.84e-04 | 5602.2ms/step | 10.7 samples/s
|
| 383 |
+
Step 6390/50462 | Loss: 2.8321 | LR: 4.84e-04 | 5601.1ms/step | 10.7 samples/s
|
| 384 |
+
Step 6400/50462 | Loss: 2.8484 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 385 |
+
Step 6410/50462 | Loss: 2.8332 | LR: 4.84e-04 | 5604.4ms/step | 10.7 samples/s
|
| 386 |
+
Step 6420/50462 | Loss: 2.8245 | LR: 4.84e-04 | 5600.4ms/step | 10.7 samples/s
|
| 387 |
+
Step 6430/50462 | Loss: 2.8550 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 388 |
+
Step 6440/50462 | Loss: 2.8666 | LR: 4.84e-04 | 5600.7ms/step | 10.7 samples/s
|
| 389 |
+
Step 6450/50462 | Loss: 2.8331 | LR: 4.84e-04 | 5603.5ms/step | 10.7 samples/s
|
| 390 |
+
Step 6460/50462 | Loss: 2.8651 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 391 |
+
Step 6470/50462 | Loss: 2.8439 | LR: 4.84e-04 | 5602.0ms/step | 10.7 samples/s
|
| 392 |
+
Step 6480/50462 | Loss: 2.8011 | LR: 4.84e-04 | 5602.5ms/step | 10.7 samples/s
|
| 393 |
+
Step 6490/50462 | Loss: 2.8101 | LR: 4.84e-04 | 5602.7ms/step | 10.7 samples/s
|
| 394 |
+
Step 6500/50462 | Loss: 2.8176 | LR: 4.84e-04 | 5601.2ms/step | 10.7 samples/s
|
| 395 |
+
Validation at step 6500: Loss = 2.8927, Perplexity = 18.04
|
| 396 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6500
|
| 397 |
+
Step 6510/50462 | Loss: 2.7805 | LR: 4.84e-04 | 8498.7ms/step | 7.1 samples/s
|
| 398 |
+
Step 6520/50462 | Loss: 2.8407 | LR: 4.84e-04 | 5599.8ms/step | 10.7 samples/s
|
| 399 |
+
Step 6530/50462 | Loss: 2.8762 | LR: 4.84e-04 | 5601.4ms/step | 10.7 samples/s
|
| 400 |
+
Step 6540/50462 | Loss: 2.8507 | LR: 4.83e-04 | 5602.0ms/step | 10.7 samples/s
|
| 401 |
+
Step 6550/50462 | Loss: 2.8411 | LR: 4.83e-04 | 5602.1ms/step | 10.7 samples/s
|
| 402 |
+
Step 6560/50462 | Loss: 2.8222 | LR: 4.83e-04 | 5602.8ms/step | 10.7 samples/s
|
| 403 |
+
Step 6570/50462 | Loss: 2.7923 | LR: 4.83e-04 | 5603.2ms/step | 10.7 samples/s
|
| 404 |
+
Step 6580/50462 | Loss: 2.8730 | LR: 4.83e-04 | 5603.4ms/step | 10.7 samples/s
|
| 405 |
+
Step 6590/50462 | Loss: 2.8510 | LR: 4.83e-04 | 5604.8ms/step | 10.7 samples/s
|
| 406 |
+
Step 6600/50462 | Loss: 2.8475 | LR: 4.83e-04 | 5602.8ms/step | 10.7 samples/s
|
| 407 |
+
Step 6610/50462 | Loss: 2.8278 | LR: 4.83e-04 | 5603.6ms/step | 10.7 samples/s
|
| 408 |
+
Step 6620/50462 | Loss: 2.8478 | LR: 4.83e-04 | 5604.3ms/step | 10.7 samples/s
|
| 409 |
+
Step 6630/50462 | Loss: 2.8333 | LR: 4.83e-04 | 5602.5ms/step | 10.7 samples/s
|
| 410 |
+
Step 6640/50462 | Loss: 2.8981 | LR: 4.83e-04 | 5601.9ms/step | 10.7 samples/s
|
| 411 |
+
Step 6650/50462 | Loss: 2.8616 | LR: 4.83e-04 | 5600.7ms/step | 10.7 samples/s
|
| 412 |
+
Step 6660/50462 | Loss: 2.7948 | LR: 4.83e-04 | 5600.7ms/step | 10.7 samples/s
|
| 413 |
+
Step 6670/50462 | Loss: 2.8245 | LR: 4.83e-04 | 5601.0ms/step | 10.7 samples/s
|
| 414 |
+
Step 6680/50462 | Loss: 2.8394 | LR: 4.83e-04 | 5603.6ms/step | 10.7 samples/s
|
| 415 |
+
Step 6690/50462 | Loss: 2.7885 | LR: 4.83e-04 | 5601.9ms/step | 10.7 samples/s
|
| 416 |
+
Step 6700/50462 | Loss: 2.8298 | LR: 4.83e-04 | 5602.5ms/step | 10.7 samples/s
|
| 417 |
+
Step 6710/50462 | Loss: 2.8079 | LR: 4.82e-04 | 5602.8ms/step | 10.7 samples/s
|
| 418 |
+
Step 6720/50462 | Loss: 2.8333 | LR: 4.82e-04 | 5603.4ms/step | 10.7 samples/s
|
| 419 |
+
Step 6730/50462 | Loss: 2.8491 | LR: 4.82e-04 | 5603.8ms/step | 10.7 samples/s
|
| 420 |
+
Step 6740/50462 | Loss: 2.8368 | LR: 4.82e-04 | 5602.5ms/step | 10.7 samples/s
|
| 421 |
+
Step 6750/50462 | Loss: 2.8223 | LR: 4.82e-04 | 5602.5ms/step | 10.7 samples/s
|
| 422 |
+
Validation at step 6750: Loss = 2.8829, Perplexity = 17.87
|
| 423 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-6750
|
| 424 |
+
Step 6760/50462 | Loss: 2.8407 | LR: 4.82e-04 | 8528.3ms/step | 7.0 samples/s
|
| 425 |
+
Step 6770/50462 | Loss: 2.7726 | LR: 4.82e-04 | 5600.9ms/step | 10.7 samples/s
|
| 426 |
+
Step 6780/50462 | Loss: 2.8296 | LR: 4.82e-04 | 5600.1ms/step | 10.7 samples/s
|
| 427 |
+
Step 6790/50462 | Loss: 2.8262 | LR: 4.82e-04 | 5602.8ms/step | 10.7 samples/s
|
| 428 |
+
Step 6800/50462 | Loss: 2.8336 | LR: 4.82e-04 | 5601.9ms/step | 10.7 samples/s
|
| 429 |
+
Step 6810/50462 | Loss: 2.8418 | LR: 4.82e-04 | 5598.7ms/step | 10.7 samples/s
|
| 430 |
+
Step 6820/50462 | Loss: 2.8416 | LR: 4.82e-04 | 5599.5ms/step | 10.7 samples/s
|
| 431 |
+
Step 6830/50462 | Loss: 2.8223 | LR: 4.82e-04 | 5598.7ms/step | 10.7 samples/s
|
| 432 |
+
Step 6840/50462 | Loss: 2.8191 | LR: 4.82e-04 | 5599.6ms/step | 10.7 samples/s
|
| 433 |
+
Step 6850/50462 | Loss: 2.8408 | LR: 4.82e-04 | 5599.3ms/step | 10.7 samples/s
|
| 434 |
+
Step 6860/50462 | Loss: 2.7948 | LR: 4.82e-04 | 5607.1ms/step | 10.7 samples/s
|
| 435 |
+
Step 6870/50462 | Loss: 2.8232 | LR: 4.82e-04 | 5602.0ms/step | 10.7 samples/s
|
| 436 |
+
Step 6880/50462 | Loss: 2.8308 | LR: 4.81e-04 | 5608.6ms/step | 10.7 samples/s
|
| 437 |
+
Step 6890/50462 | Loss: 2.8011 | LR: 4.81e-04 | 5612.1ms/step | 10.7 samples/s
|
| 438 |
+
Step 6900/50462 | Loss: 2.7990 | LR: 4.81e-04 | 5608.8ms/step | 10.7 samples/s
|
| 439 |
+
Step 6910/50462 | Loss: 2.8602 | LR: 4.81e-04 | 5609.3ms/step | 10.7 samples/s
|
| 440 |
+
Step 6920/50462 | Loss: 2.8411 | LR: 4.81e-04 | 5610.0ms/step | 10.7 samples/s
|
| 441 |
+
Step 6930/50462 | Loss: 2.8294 | LR: 4.81e-04 | 5611.2ms/step | 10.7 samples/s
|
| 442 |
+
Step 6940/50462 | Loss: 2.8712 | LR: 4.81e-04 | 5611.2ms/step | 10.7 samples/s
|
| 443 |
+
Step 6950/50462 | Loss: 2.8333 | LR: 4.81e-04 | 5607.3ms/step | 10.7 samples/s
|
| 444 |
+
Step 6960/50462 | Loss: 2.8152 | LR: 4.81e-04 | 5608.0ms/step | 10.7 samples/s
|
| 445 |
+
Step 6970/50462 | Loss: 2.8187 | LR: 4.81e-04 | 5608.8ms/step | 10.7 samples/s
|
| 446 |
+
Step 6980/50462 | Loss: 2.8442 | LR: 4.81e-04 | 5610.3ms/step | 10.7 samples/s
|
| 447 |
+
Step 6990/50462 | Loss: 2.8114 | LR: 4.81e-04 | 5608.5ms/step | 10.7 samples/s
|
| 448 |
+
Step 7000/50462 | Loss: 2.8674 | LR: 4.81e-04 | 5606.9ms/step | 10.7 samples/s
|
| 449 |
+
Validation at step 7000: Loss = 2.8742, Perplexity = 17.71
|
| 450 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7000
|
| 451 |
+
Step 7010/50462 | Loss: 2.8103 | LR: 4.81e-04 | 8484.8ms/step | 7.1 samples/s
|
| 452 |
+
Step 7020/50462 | Loss: 2.7829 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 453 |
+
Step 7030/50462 | Loss: 2.7681 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 454 |
+
Step 7040/50462 | Loss: 2.8460 | LR: 4.81e-04 | 5606.5ms/step | 10.7 samples/s
|
| 455 |
+
Step 7050/50462 | Loss: 2.7953 | LR: 4.80e-04 | 5606.3ms/step | 10.7 samples/s
|
| 456 |
+
Step 7060/50462 | Loss: 2.8414 | LR: 4.80e-04 | 5604.6ms/step | 10.7 samples/s
|
| 457 |
+
Step 7070/50462 | Loss: 2.8295 | LR: 4.80e-04 | 5607.4ms/step | 10.7 samples/s
|
| 458 |
+
Step 7080/50462 | Loss: 2.8229 | LR: 4.80e-04 | 5609.0ms/step | 10.7 samples/s
|
| 459 |
+
Step 7090/50462 | Loss: 2.8258 | LR: 4.80e-04 | 5607.6ms/step | 10.7 samples/s
|
| 460 |
+
Step 7100/50462 | Loss: 2.8283 | LR: 4.80e-04 | 5607.3ms/step | 10.7 samples/s
|
| 461 |
+
Step 7110/50462 | Loss: 2.8068 | LR: 4.80e-04 | 5605.6ms/step | 10.7 samples/s
|
| 462 |
+
Step 7120/50462 | Loss: 2.8171 | LR: 4.80e-04 | 5608.4ms/step | 10.7 samples/s
|
| 463 |
+
Step 7130/50462 | Loss: 2.8284 | LR: 4.80e-04 | 5609.2ms/step | 10.7 samples/s
|
| 464 |
+
Step 7140/50462 | Loss: 2.7921 | LR: 4.80e-04 | 5607.4ms/step | 10.7 samples/s
|
| 465 |
+
Step 7150/50462 | Loss: 2.8284 | LR: 4.80e-04 | 5605.7ms/step | 10.7 samples/s
|
| 466 |
+
Step 7160/50462 | Loss: 2.7888 | LR: 4.80e-04 | 5608.1ms/step | 10.7 samples/s
|
| 467 |
+
Step 7170/50462 | Loss: 2.8640 | LR: 4.80e-04 | 5606.7ms/step | 10.7 samples/s
|
| 468 |
+
Step 7180/50462 | Loss: 2.7964 | LR: 4.80e-04 | 5604.3ms/step | 10.7 samples/s
|
| 469 |
+
Step 7190/50462 | Loss: 2.8355 | LR: 4.80e-04 | 5606.2ms/step | 10.7 samples/s
|
| 470 |
+
Step 7200/50462 | Loss: 2.8251 | LR: 4.80e-04 | 5605.8ms/step | 10.7 samples/s
|
| 471 |
+
Step 7210/50462 | Loss: 2.8124 | LR: 4.79e-04 | 5606.9ms/step | 10.7 samples/s
|
| 472 |
+
Step 7220/50462 | Loss: 2.8326 | LR: 4.79e-04 | 5607.1ms/step | 10.7 samples/s
|
| 473 |
+
Step 7230/50462 | Loss: 2.8360 | LR: 4.79e-04 | 5607.2ms/step | 10.7 samples/s
|
| 474 |
+
Step 7240/50462 | Loss: 2.7862 | LR: 4.79e-04 | 5604.5ms/step | 10.7 samples/s
|
| 475 |
+
Step 7250/50462 | Loss: 2.8325 | LR: 4.79e-04 | 5607.8ms/step | 10.7 samples/s
|
| 476 |
+
Validation at step 7250: Loss = 2.8687, Perplexity = 17.61
|
| 477 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7250
|
| 478 |
+
Step 7260/50462 | Loss: 2.8578 | LR: 4.79e-04 | 8405.5ms/step | 7.1 samples/s
|
| 479 |
+
Step 7270/50462 | Loss: 2.8250 | LR: 4.79e-04 | 5602.0ms/step | 10.7 samples/s
|
| 480 |
+
Step 7280/50462 | Loss: 2.8292 | LR: 4.79e-04 | 5604.6ms/step | 10.7 samples/s
|
| 481 |
+
Step 7290/50462 | Loss: 2.8172 | LR: 4.79e-04 | 5605.3ms/step | 10.7 samples/s
|
| 482 |
+
Step 7300/50462 | Loss: 2.7778 | LR: 4.79e-04 | 5606.0ms/step | 10.7 samples/s
|
| 483 |
+
Step 7310/50462 | Loss: 2.8234 | LR: 4.79e-04 | 5607.1ms/step | 10.7 samples/s
|
| 484 |
+
Step 7320/50462 | Loss: 2.8029 | LR: 4.79e-04 | 5609.3ms/step | 10.7 samples/s
|
| 485 |
+
Step 7330/50462 | Loss: 2.8036 | LR: 4.79e-04 | 5607.5ms/step | 10.7 samples/s
|
| 486 |
+
Step 7340/50462 | Loss: 2.8200 | LR: 4.79e-04 | 5605.8ms/step | 10.7 samples/s
|
| 487 |
+
Step 7350/50462 | Loss: 2.7972 | LR: 4.79e-04 | 5606.6ms/step | 10.7 samples/s
|
| 488 |
+
Step 7360/50462 | Loss: 2.7684 | LR: 4.79e-04 | 5607.0ms/step | 10.7 samples/s
|
| 489 |
+
Step 7370/50462 | Loss: 2.8106 | LR: 4.78e-04 | 5606.2ms/step | 10.7 samples/s
|
| 490 |
+
Step 7380/50462 | Loss: 2.7749 | LR: 4.78e-04 | 5607.3ms/step | 10.7 samples/s
|
| 491 |
+
Step 7390/50462 | Loss: 2.8407 | LR: 4.78e-04 | 5607.1ms/step | 10.7 samples/s
|
| 492 |
+
Step 7400/50462 | Loss: 2.8236 | LR: 4.78e-04 | 5607.3ms/step | 10.7 samples/s
|
| 493 |
+
Step 7410/50462 | Loss: 2.8280 | LR: 4.78e-04 | 5604.0ms/step | 10.7 samples/s
|
| 494 |
+
Step 7420/50462 | Loss: 2.8239 | LR: 4.78e-04 | 5605.8ms/step | 10.7 samples/s
|
| 495 |
+
Step 7430/50462 | Loss: 2.7875 | LR: 4.78e-04 | 5610.1ms/step | 10.7 samples/s
|
| 496 |
+
Step 7440/50462 | Loss: 2.8026 | LR: 4.78e-04 | 5607.4ms/step | 10.7 samples/s
|
| 497 |
+
Step 7450/50462 | Loss: 2.8018 | LR: 4.78e-04 | 5605.3ms/step | 10.7 samples/s
|
| 498 |
+
Step 7460/50462 | Loss: 2.8252 | LR: 4.78e-04 | 5608.0ms/step | 10.7 samples/s
|
| 499 |
+
Step 7470/50462 | Loss: 2.7949 | LR: 4.78e-04 | 5605.8ms/step | 10.7 samples/s
|
| 500 |
+
Step 7480/50462 | Loss: 2.8329 | LR: 4.78e-04 | 5607.9ms/step | 10.7 samples/s
|
| 501 |
+
Step 7490/50462 | Loss: 2.7891 | LR: 4.78e-04 | 5606.8ms/step | 10.7 samples/s
|
| 502 |
+
Step 7500/50462 | Loss: 2.7779 | LR: 4.78e-04 | 5603.6ms/step | 10.7 samples/s
|
| 503 |
+
Validation at step 7500: Loss = 2.8601, Perplexity = 17.46
|
| 504 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7500
|
| 505 |
+
Step 7510/50462 | Loss: 2.8206 | LR: 4.78e-04 | 8499.2ms/step | 7.1 samples/s
|
| 506 |
+
Step 7520/50462 | Loss: 2.8255 | LR: 4.77e-04 | 5596.4ms/step | 10.7 samples/s
|
| 507 |
+
Step 7530/50462 | Loss: 2.8260 | LR: 4.77e-04 | 5602.2ms/step | 10.7 samples/s
|
| 508 |
+
Step 7540/50462 | Loss: 2.7634 | LR: 4.77e-04 | 5605.4ms/step | 10.7 samples/s
|
| 509 |
+
Step 7550/50462 | Loss: 2.7544 | LR: 4.77e-04 | 5605.7ms/step | 10.7 samples/s
|
| 510 |
+
Step 7560/50462 | Loss: 2.8360 | LR: 4.77e-04 | 5604.6ms/step | 10.7 samples/s
|
| 511 |
+
Step 7570/50462 | Loss: 2.7680 | LR: 4.77e-04 | 5613.9ms/step | 10.7 samples/s
|
| 512 |
+
Step 7580/50462 | Loss: 2.7691 | LR: 4.77e-04 | 5609.5ms/step | 10.7 samples/s
|
| 513 |
+
Step 7590/50462 | Loss: 2.7912 | LR: 4.77e-04 | 5609.0ms/step | 10.7 samples/s
|
| 514 |
+
Step 7600/50462 | Loss: 2.8269 | LR: 4.77e-04 | 5610.9ms/step | 10.7 samples/s
|
| 515 |
+
Step 7610/50462 | Loss: 2.8439 | LR: 4.77e-04 | 5613.6ms/step | 10.7 samples/s
|
| 516 |
+
Step 7620/50462 | Loss: 2.8021 | LR: 4.77e-04 | 5615.1ms/step | 10.7 samples/s
|
| 517 |
+
Step 7630/50462 | Loss: 2.8128 | LR: 4.77e-04 | 5608.9ms/step | 10.7 samples/s
|
| 518 |
+
Step 7640/50462 | Loss: 2.8043 | LR: 4.77e-04 | 5616.4ms/step | 10.7 samples/s
|
| 519 |
+
Step 7650/50462 | Loss: 2.8083 | LR: 4.77e-04 | 5630.2ms/step | 10.7 samples/s
|
| 520 |
+
Step 7660/50462 | Loss: 2.7890 | LR: 4.77e-04 | 5617.2ms/step | 10.7 samples/s
|
| 521 |
+
Step 7670/50462 | Loss: 2.7772 | LR: 4.76e-04 | 5607.5ms/step | 10.7 samples/s
|
| 522 |
+
Step 7680/50462 | Loss: 2.7793 | LR: 4.76e-04 | 5610.5ms/step | 10.7 samples/s
|
| 523 |
+
Step 7690/50462 | Loss: 2.8036 | LR: 4.76e-04 | 5620.3ms/step | 10.7 samples/s
|
| 524 |
+
Step 7700/50462 | Loss: 2.7944 | LR: 4.76e-04 | 5610.1ms/step | 10.7 samples/s
|
| 525 |
+
Step 7710/50462 | Loss: 2.7520 | LR: 4.76e-04 | 5610.0ms/step | 10.7 samples/s
|
| 526 |
+
Step 7720/50462 | Loss: 2.8115 | LR: 4.76e-04 | 5608.5ms/step | 10.7 samples/s
|
| 527 |
+
Step 7730/50462 | Loss: 2.8230 | LR: 4.76e-04 | 5609.3ms/step | 10.7 samples/s
|
| 528 |
+
Step 7740/50462 | Loss: 2.8089 | LR: 4.76e-04 | 5608.4ms/step | 10.7 samples/s
|
| 529 |
+
Step 7750/50462 | Loss: 2.7953 | LR: 4.76e-04 | 5607.9ms/step | 10.7 samples/s
|
| 530 |
+
Validation at step 7750: Loss = 2.8521, Perplexity = 17.32
|
| 531 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-7750
|
| 532 |
+
Step 7760/50462 | Loss: 2.8306 | LR: 4.76e-04 | 8331.5ms/step | 7.2 samples/s
|
| 533 |
+
Step 7770/50462 | Loss: 2.8193 | LR: 4.76e-04 | 5599.0ms/step | 10.7 samples/s
|
| 534 |
+
Step 7780/50462 | Loss: 2.7795 | LR: 4.76e-04 | 5605.1ms/step | 10.7 samples/s
|
| 535 |
+
Step 7790/50462 | Loss: 2.7692 | LR: 4.76e-04 | 5607.7ms/step | 10.7 samples/s
|
| 536 |
+
Step 7800/50462 | Loss: 2.7805 | LR: 4.76e-04 | 5607.5ms/step | 10.7 samples/s
|
| 537 |
+
Step 7810/50462 | Loss: 2.8043 | LR: 4.76e-04 | 5605.2ms/step | 10.7 samples/s
|
| 538 |
+
Step 7820/50462 | Loss: 2.7824 | LR: 4.75e-04 | 5610.0ms/step | 10.7 samples/s
|
| 539 |
+
Step 7830/50462 | Loss: 2.7889 | LR: 4.75e-04 | 5607.2ms/step | 10.7 samples/s
|
| 540 |
+
Step 7840/50462 | Loss: 2.8430 | LR: 4.75e-04 | 5612.5ms/step | 10.7 samples/s
|
| 541 |
+
Step 7850/50462 | Loss: 2.7800 | LR: 4.75e-04 | 5612.6ms/step | 10.7 samples/s
|
| 542 |
+
Step 7860/50462 | Loss: 2.8191 | LR: 4.75e-04 | 5609.4ms/step | 10.7 samples/s
|
| 543 |
+
Step 7870/50462 | Loss: 2.8351 | LR: 4.75e-04 | 5608.0ms/step | 10.7 samples/s
|
| 544 |
+
Step 7880/50462 | Loss: 2.7958 | LR: 4.75e-04 | 5607.2ms/step | 10.7 samples/s
|
| 545 |
+
Step 7890/50462 | Loss: 2.8373 | LR: 4.75e-04 | 5609.8ms/step | 10.7 samples/s
|
| 546 |
+
Step 7900/50462 | Loss: 2.8238 | LR: 4.75e-04 | 5614.4ms/step | 10.7 samples/s
|
| 547 |
+
Step 7910/50462 | Loss: 2.7748 | LR: 4.75e-04 | 5605.2ms/step | 10.7 samples/s
|
| 548 |
+
Step 7920/50462 | Loss: 2.7886 | LR: 4.75e-04 | 5609.6ms/step | 10.7 samples/s
|
| 549 |
+
Step 7930/50462 | Loss: 2.7858 | LR: 4.75e-04 | 5609.4ms/step | 10.7 samples/s
|
| 550 |
+
Step 7940/50462 | Loss: 2.8201 | LR: 4.75e-04 | 5608.5ms/step | 10.7 samples/s
|
| 551 |
+
Step 7950/50462 | Loss: 2.8178 | LR: 4.75e-04 | 5617.4ms/step | 10.7 samples/s
|
| 552 |
+
Step 7960/50462 | Loss: 2.8199 | LR: 4.74e-04 | 5618.3ms/step | 10.7 samples/s
|
| 553 |
+
Step 7970/50462 | Loss: 2.8284 | LR: 4.74e-04 | 5609.5ms/step | 10.7 samples/s
|
| 554 |
+
Step 7980/50462 | Loss: 2.8000 | LR: 4.74e-04 | 5613.1ms/step | 10.7 samples/s
|
| 555 |
+
Step 7990/50462 | Loss: 2.7719 | LR: 4.74e-04 | 5619.0ms/step | 10.7 samples/s
|
| 556 |
+
Step 8000/50462 | Loss: 2.8038 | LR: 4.74e-04 | 5615.3ms/step | 10.7 samples/s
|
| 557 |
+
Validation at step 8000: Loss = 2.8459, Perplexity = 17.22
|
| 558 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8000
|
| 559 |
+
Step 8010/50462 | Loss: 2.7816 | LR: 4.74e-04 | 8393.4ms/step | 7.1 samples/s
|
| 560 |
+
Step 8020/50462 | Loss: 2.7759 | LR: 4.74e-04 | 5607.4ms/step | 10.7 samples/s
|
| 561 |
+
Step 8030/50462 | Loss: 2.7861 | LR: 4.74e-04 | 5611.8ms/step | 10.7 samples/s
|
| 562 |
+
Step 8040/50462 | Loss: 2.7721 | LR: 4.74e-04 | 5614.7ms/step | 10.7 samples/s
|
| 563 |
+
Step 8050/50462 | Loss: 2.7873 | LR: 4.74e-04 | 5609.6ms/step | 10.7 samples/s
|
| 564 |
+
Step 8060/50462 | Loss: 2.7801 | LR: 4.74e-04 | 5615.1ms/step | 10.7 samples/s
|
| 565 |
+
Step 8070/50462 | Loss: 2.8520 | LR: 4.74e-04 | 5614.2ms/step | 10.7 samples/s
|
| 566 |
+
Step 8080/50462 | Loss: 2.7732 | LR: 4.74e-04 | 5617.2ms/step | 10.7 samples/s
|
| 567 |
+
Step 8090/50462 | Loss: 2.7643 | LR: 4.74e-04 | 5613.3ms/step | 10.7 samples/s
|
| 568 |
+
Step 8100/50462 | Loss: 2.8193 | LR: 4.74e-04 | 5615.8ms/step | 10.7 samples/s
|
| 569 |
+
Step 8110/50462 | Loss: 2.7653 | LR: 4.73e-04 | 5613.5ms/step | 10.7 samples/s
|
| 570 |
+
Step 8120/50462 | Loss: 2.7890 | LR: 4.73e-04 | 5615.0ms/step | 10.7 samples/s
|
| 571 |
+
Step 8130/50462 | Loss: 2.7724 | LR: 4.73e-04 | 5613.0ms/step | 10.7 samples/s
|
| 572 |
+
Step 8140/50462 | Loss: 2.7874 | LR: 4.73e-04 | 5612.8ms/step | 10.7 samples/s
|
| 573 |
+
Step 8150/50462 | Loss: 2.7921 | LR: 4.73e-04 | 5609.4ms/step | 10.7 samples/s
|
| 574 |
+
Step 8160/50462 | Loss: 2.8130 | LR: 4.73e-04 | 5609.0ms/step | 10.7 samples/s
|
| 575 |
+
Step 8170/50462 | Loss: 2.8034 | LR: 4.73e-04 | 5610.4ms/step | 10.7 samples/s
|
| 576 |
+
Step 8180/50462 | Loss: 2.8194 | LR: 4.73e-04 | 5611.1ms/step | 10.7 samples/s
|
| 577 |
+
Step 8190/50462 | Loss: 2.7971 | LR: 4.73e-04 | 5608.1ms/step | 10.7 samples/s
|
| 578 |
+
Step 8200/50462 | Loss: 2.8226 | LR: 4.73e-04 | 5608.1ms/step | 10.7 samples/s
|
| 579 |
+
Step 8210/50462 | Loss: 2.7842 | LR: 4.73e-04 | 5608.2ms/step | 10.7 samples/s
|
| 580 |
+
Step 8220/50462 | Loss: 2.7515 | LR: 4.73e-04 | 5608.8ms/step | 10.7 samples/s
|
| 581 |
+
Step 8230/50462 | Loss: 2.7711 | LR: 4.73e-04 | 5606.8ms/step | 10.7 samples/s
|
| 582 |
+
Step 8240/50462 | Loss: 2.8089 | LR: 4.73e-04 | 5608.8ms/step | 10.7 samples/s
|
| 583 |
+
Step 8250/50462 | Loss: 2.7797 | LR: 4.72e-04 | 5609.6ms/step | 10.7 samples/s
|
| 584 |
+
Validation at step 8250: Loss = 2.8379, Perplexity = 17.08
|
| 585 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8250
|
| 586 |
+
Step 8260/50462 | Loss: 2.8130 | LR: 4.72e-04 | 8335.4ms/step | 7.2 samples/s
|
| 587 |
+
Step 8270/50462 | Loss: 2.7933 | LR: 4.72e-04 | 5602.1ms/step | 10.7 samples/s
|
| 588 |
+
Step 8280/50462 | Loss: 2.7686 | LR: 4.72e-04 | 5606.8ms/step | 10.7 samples/s
|
| 589 |
+
Step 8290/50462 | Loss: 2.7837 | LR: 4.72e-04 | 5606.9ms/step | 10.7 samples/s
|
| 590 |
+
Step 8300/50462 | Loss: 2.7548 | LR: 4.72e-04 | 5606.6ms/step | 10.7 samples/s
|
| 591 |
+
Step 8310/50462 | Loss: 2.8076 | LR: 4.72e-04 | 5608.0ms/step | 10.7 samples/s
|
| 592 |
+
Step 8320/50462 | Loss: 2.7677 | LR: 4.72e-04 | 5608.8ms/step | 10.7 samples/s
|
| 593 |
+
Step 8330/50462 | Loss: 2.7699 | LR: 4.72e-04 | 5609.2ms/step | 10.7 samples/s
|
| 594 |
+
Step 8340/50462 | Loss: 2.8124 | LR: 4.72e-04 | 5607.6ms/step | 10.7 samples/s
|
| 595 |
+
Step 8350/50462 | Loss: 2.7834 | LR: 4.72e-04 | 5606.7ms/step | 10.7 samples/s
|
| 596 |
+
Step 8360/50462 | Loss: 2.8255 | LR: 4.72e-04 | 5607.4ms/step | 10.7 samples/s
|
| 597 |
+
Step 8370/50462 | Loss: 2.7844 | LR: 4.72e-04 | 5605.1ms/step | 10.7 samples/s
|
| 598 |
+
Step 8380/50462 | Loss: 2.7884 | LR: 4.71e-04 | 5607.2ms/step | 10.7 samples/s
|
| 599 |
+
Step 8390/50462 | Loss: 2.7772 | LR: 4.71e-04 | 5608.0ms/step | 10.7 samples/s
|
| 600 |
+
Step 8400/50462 | Loss: 2.7743 | LR: 4.71e-04 | 5606.6ms/step | 10.7 samples/s
|
| 601 |
+
Step 8410/50462 | Loss: 2.7925 | LR: 4.71e-04 | 5606.6ms/step | 10.7 samples/s
|
| 602 |
+
Step 8420/50462 | Loss: 2.8248 | LR: 4.71e-04 | 5607.0ms/step | 10.7 samples/s
|
| 603 |
+
Step 8430/50462 | Loss: 2.7775 | LR: 4.71e-04 | 5606.2ms/step | 10.7 samples/s
|
| 604 |
+
Step 8440/50462 | Loss: 2.7937 | LR: 4.71e-04 | 5607.1ms/step | 10.7 samples/s
|
| 605 |
+
Step 8450/50462 | Loss: 2.7623 | LR: 4.71e-04 | 5606.1ms/step | 10.7 samples/s
|
| 606 |
+
Step 8460/50462 | Loss: 2.7462 | LR: 4.71e-04 | 5608.2ms/step | 10.7 samples/s
|
| 607 |
+
Step 8470/50462 | Loss: 2.8054 | LR: 4.71e-04 | 5614.4ms/step | 10.7 samples/s
|
| 608 |
+
Step 8480/50462 | Loss: 2.7772 | LR: 4.71e-04 | 5615.1ms/step | 10.7 samples/s
|
| 609 |
+
Step 8490/50462 | Loss: 2.7954 | LR: 4.71e-04 | 5609.9ms/step | 10.7 samples/s
|
| 610 |
+
Step 8500/50462 | Loss: 2.7854 | LR: 4.71e-04 | 5612.6ms/step | 10.7 samples/s
|
| 611 |
+
Validation at step 8500: Loss = 2.8318, Perplexity = 16.98
|
| 612 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8500
|
| 613 |
+
Step 8510/50462 | Loss: 2.7415 | LR: 4.71e-04 | 8383.3ms/step | 7.2 samples/s
|
| 614 |
+
Step 8520/50462 | Loss: 2.7724 | LR: 4.70e-04 | 5607.3ms/step | 10.7 samples/s
|
| 615 |
+
Step 8530/50462 | Loss: 2.7970 | LR: 4.70e-04 | 5611.9ms/step | 10.7 samples/s
|
| 616 |
+
Step 8540/50462 | Loss: 2.7206 | LR: 4.70e-04 | 5610.2ms/step | 10.7 samples/s
|
| 617 |
+
Step 8550/50462 | Loss: 2.7360 | LR: 4.70e-04 | 5611.6ms/step | 10.7 samples/s
|
| 618 |
+
Step 8560/50462 | Loss: 2.7586 | LR: 4.70e-04 | 5613.6ms/step | 10.7 samples/s
|
| 619 |
+
Step 8570/50462 | Loss: 2.7953 | LR: 4.70e-04 | 5614.1ms/step | 10.7 samples/s
|
| 620 |
+
Step 8580/50462 | Loss: 2.7844 | LR: 4.70e-04 | 5613.4ms/step | 10.7 samples/s
|
| 621 |
+
Step 8590/50462 | Loss: 2.7743 | LR: 4.70e-04 | 5611.3ms/step | 10.7 samples/s
|
| 622 |
+
Step 8600/50462 | Loss: 2.7710 | LR: 4.70e-04 | 5614.2ms/step | 10.7 samples/s
|
| 623 |
+
Step 8610/50462 | Loss: 2.7338 | LR: 4.70e-04 | 5614.5ms/step | 10.7 samples/s
|
| 624 |
+
Step 8620/50462 | Loss: 2.7743 | LR: 4.70e-04 | 5616.0ms/step | 10.7 samples/s
|
| 625 |
+
Step 8630/50462 | Loss: 2.7633 | LR: 4.70e-04 | 5616.2ms/step | 10.7 samples/s
|
| 626 |
+
Step 8640/50462 | Loss: 2.7525 | LR: 4.70e-04 | 5614.0ms/step | 10.7 samples/s
|
| 627 |
+
Step 8650/50462 | Loss: 2.7807 | LR: 4.69e-04 | 5615.4ms/step | 10.7 samples/s
|
| 628 |
+
Step 8660/50462 | Loss: 2.7697 | LR: 4.69e-04 | 5617.6ms/step | 10.7 samples/s
|
| 629 |
+
Step 8670/50462 | Loss: 2.7909 | LR: 4.69e-04 | 5615.0ms/step | 10.7 samples/s
|
| 630 |
+
Step 8680/50462 | Loss: 2.7759 | LR: 4.69e-04 | 5616.2ms/step | 10.7 samples/s
|
| 631 |
+
Step 8690/50462 | Loss: 2.7675 | LR: 4.69e-04 | 5619.6ms/step | 10.7 samples/s
|
| 632 |
+
Step 8700/50462 | Loss: 2.8104 | LR: 4.69e-04 | 5618.5ms/step | 10.7 samples/s
|
| 633 |
+
Step 8710/50462 | Loss: 2.7951 | LR: 4.69e-04 | 5613.6ms/step | 10.7 samples/s
|
| 634 |
+
Step 8720/50462 | Loss: 2.7897 | LR: 4.69e-04 | 5616.5ms/step | 10.7 samples/s
|
| 635 |
+
Step 8730/50462 | Loss: 2.7354 | LR: 4.69e-04 | 5615.2ms/step | 10.7 samples/s
|
| 636 |
+
Step 8740/50462 | Loss: 2.7694 | LR: 4.69e-04 | 5615.6ms/step | 10.7 samples/s
|
| 637 |
+
Step 8750/50462 | Loss: 2.8096 | LR: 4.69e-04 | 5617.2ms/step | 10.7 samples/s
|
| 638 |
+
Validation at step 8750: Loss = 2.8289, Perplexity = 16.93
|
| 639 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-8750
|
| 640 |
+
Step 8760/50462 | Loss: 2.7627 | LR: 4.69e-04 | 8396.2ms/step | 7.1 samples/s
|
| 641 |
+
Step 8770/50462 | Loss: 2.7783 | LR: 4.69e-04 | 5602.9ms/step | 10.7 samples/s
|
| 642 |
+
Step 8780/50462 | Loss: 2.7793 | LR: 4.68e-04 | 5604.7ms/step | 10.7 samples/s
|
| 643 |
+
Step 8790/50462 | Loss: 2.7471 | LR: 4.68e-04 | 5604.8ms/step | 10.7 samples/s
|
| 644 |
+
Step 8800/50462 | Loss: 2.7721 | LR: 4.68e-04 | 5606.9ms/step | 10.7 samples/s
|
| 645 |
+
Step 8810/50462 | Loss: 2.8129 | LR: 4.68e-04 | 5606.7ms/step | 10.7 samples/s
|
| 646 |
+
Step 8820/50462 | Loss: 2.7578 | LR: 4.68e-04 | 5605.5ms/step | 10.7 samples/s
|
| 647 |
+
Step 8830/50462 | Loss: 2.8022 | LR: 4.68e-04 | 5604.8ms/step | 10.7 samples/s
|
| 648 |
+
Step 8840/50462 | Loss: 2.8083 | LR: 4.68e-04 | 5609.0ms/step | 10.7 samples/s
|
| 649 |
+
Step 8850/50462 | Loss: 2.7074 | LR: 4.68e-04 | 5608.0ms/step | 10.7 samples/s
|
| 650 |
+
Step 8860/50462 | Loss: 2.7886 | LR: 4.68e-04 | 5607.5ms/step | 10.7 samples/s
|
| 651 |
+
Step 8870/50462 | Loss: 2.7390 | LR: 4.68e-04 | 5606.7ms/step | 10.7 samples/s
|
| 652 |
+
Step 8880/50462 | Loss: 2.7719 | LR: 4.68e-04 | 5613.8ms/step | 10.7 samples/s
|
| 653 |
+
Step 8890/50462 | Loss: 2.7811 | LR: 4.68e-04 | 5611.0ms/step | 10.7 samples/s
|
| 654 |
+
Step 8900/50462 | Loss: 2.7505 | LR: 4.68e-04 | 5606.0ms/step | 10.7 samples/s
|
| 655 |
+
Step 8910/50462 | Loss: 2.7700 | LR: 4.67e-04 | 5607.6ms/step | 10.7 samples/s
|
| 656 |
+
Step 8920/50462 | Loss: 2.7748 | LR: 4.67e-04 | 5607.2ms/step | 10.7 samples/s
|
| 657 |
+
Step 8930/50462 | Loss: 2.7463 | LR: 4.67e-04 | 5607.8ms/step | 10.7 samples/s
|
| 658 |
+
Step 8940/50462 | Loss: 2.7942 | LR: 4.67e-04 | 5605.7ms/step | 10.7 samples/s
|
| 659 |
+
Step 8950/50462 | Loss: 2.7552 | LR: 4.67e-04 | 5608.3ms/step | 10.7 samples/s
|
| 660 |
+
Step 8960/50462 | Loss: 2.7496 | LR: 4.67e-04 | 5605.8ms/step | 10.7 samples/s
|
| 661 |
+
Step 8970/50462 | Loss: 2.7428 | LR: 4.67e-04 | 5606.2ms/step | 10.7 samples/s
|
| 662 |
+
Step 8980/50462 | Loss: 2.8126 | LR: 4.67e-04 | 5608.2ms/step | 10.7 samples/s
|
| 663 |
+
Step 8990/50462 | Loss: 2.7853 | LR: 4.67e-04 | 5605.9ms/step | 10.7 samples/s
|
| 664 |
+
Step 9000/50462 | Loss: 2.7523 | LR: 4.67e-04 | 5607.3ms/step | 10.7 samples/s
|
| 665 |
+
Validation at step 9000: Loss = 2.8188, Perplexity = 16.76
|
| 666 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9000
|
| 667 |
+
Step 9010/50462 | Loss: 2.7694 | LR: 4.67e-04 | 8490.8ms/step | 7.1 samples/s
|
| 668 |
+
Step 9020/50462 | Loss: 2.7579 | LR: 4.67e-04 | 5609.8ms/step | 10.7 samples/s
|
| 669 |
+
Step 9030/50462 | Loss: 2.7669 | LR: 4.67e-04 | 5610.7ms/step | 10.7 samples/s
|
| 670 |
+
Step 9040/50462 | Loss: 2.7845 | LR: 4.66e-04 | 5613.3ms/step | 10.7 samples/s
|
| 671 |
+
Step 9050/50462 | Loss: 2.7730 | LR: 4.66e-04 | 5612.6ms/step | 10.7 samples/s
|
| 672 |
+
Step 9060/50462 | Loss: 2.7563 | LR: 4.66e-04 | 5612.0ms/step | 10.7 samples/s
|
| 673 |
+
Step 9070/50462 | Loss: 2.7574 | LR: 4.66e-04 | 5611.3ms/step | 10.7 samples/s
|
| 674 |
+
Step 9080/50462 | Loss: 2.7771 | LR: 4.66e-04 | 5609.7ms/step | 10.7 samples/s
|
| 675 |
+
Step 9090/50462 | Loss: 2.7305 | LR: 4.66e-04 | 5611.8ms/step | 10.7 samples/s
|
| 676 |
+
Step 9100/50462 | Loss: 2.7742 | LR: 4.66e-04 | 5612.3ms/step | 10.7 samples/s
|
| 677 |
+
Step 9110/50462 | Loss: 2.7700 | LR: 4.66e-04 | 5610.0ms/step | 10.7 samples/s
|
| 678 |
+
Step 9120/50462 | Loss: 2.7834 | LR: 4.66e-04 | 5610.3ms/step | 10.7 samples/s
|
| 679 |
+
Step 9130/50462 | Loss: 2.7510 | LR: 4.66e-04 | 5612.1ms/step | 10.7 samples/s
|
| 680 |
+
Step 9140/50462 | Loss: 2.7705 | LR: 4.66e-04 | 5610.0ms/step | 10.7 samples/s
|
| 681 |
+
Step 9150/50462 | Loss: 2.8000 | LR: 4.66e-04 | 5611.5ms/step | 10.7 samples/s
|
| 682 |
+
Step 9160/50462 | Loss: 2.7886 | LR: 4.66e-04 | 5613.0ms/step | 10.7 samples/s
|
| 683 |
+
Step 9170/50462 | Loss: 2.7411 | LR: 4.65e-04 | 5615.7ms/step | 10.7 samples/s
|
| 684 |
+
Step 9180/50462 | Loss: 2.7651 | LR: 4.65e-04 | 5609.5ms/step | 10.7 samples/s
|
| 685 |
+
Step 9190/50462 | Loss: 2.7721 | LR: 4.65e-04 | 5609.0ms/step | 10.7 samples/s
|
| 686 |
+
Step 9200/50462 | Loss: 2.7539 | LR: 4.65e-04 | 5610.0ms/step | 10.7 samples/s
|
| 687 |
+
Step 9210/50462 | Loss: 2.7694 | LR: 4.65e-04 | 5609.8ms/step | 10.7 samples/s
|
| 688 |
+
Step 9220/50462 | Loss: 2.7531 | LR: 4.65e-04 | 5613.4ms/step | 10.7 samples/s
|
| 689 |
+
Step 9230/50462 | Loss: 2.7699 | LR: 4.65e-04 | 5612.8ms/step | 10.7 samples/s
|
| 690 |
+
Step 9240/50462 | Loss: 2.7466 | LR: 4.65e-04 | 5610.0ms/step | 10.7 samples/s
|
| 691 |
+
Step 9250/50462 | Loss: 2.7779 | LR: 4.65e-04 | 5611.1ms/step | 10.7 samples/s
|
| 692 |
+
Validation at step 9250: Loss = 2.8126, Perplexity = 16.65
|
| 693 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9250
|
| 694 |
+
Step 9260/50462 | Loss: 2.7286 | LR: 4.65e-04 | 8343.4ms/step | 7.2 samples/s
|
| 695 |
+
Step 9270/50462 | Loss: 2.7800 | LR: 4.65e-04 | 5605.4ms/step | 10.7 samples/s
|
| 696 |
+
Step 9280/50462 | Loss: 2.7675 | LR: 4.65e-04 | 5608.0ms/step | 10.7 samples/s
|
| 697 |
+
Step 9290/50462 | Loss: 2.7647 | LR: 4.64e-04 | 5607.5ms/step | 10.7 samples/s
|
| 698 |
+
Step 9300/50462 | Loss: 2.7586 | LR: 4.64e-04 | 5608.3ms/step | 10.7 samples/s
|
| 699 |
+
Step 9310/50462 | Loss: 2.7486 | LR: 4.64e-04 | 5609.4ms/step | 10.7 samples/s
|
| 700 |
+
Step 9320/50462 | Loss: 2.7386 | LR: 4.64e-04 | 5607.4ms/step | 10.7 samples/s
|
| 701 |
+
Step 9330/50462 | Loss: 2.7595 | LR: 4.64e-04 | 5607.2ms/step | 10.7 samples/s
|
| 702 |
+
Step 9340/50462 | Loss: 2.7449 | LR: 4.64e-04 | 5606.1ms/step | 10.7 samples/s
|
| 703 |
+
Step 9350/50462 | Loss: 2.7588 | LR: 4.64e-04 | 5608.8ms/step | 10.7 samples/s
|
| 704 |
+
Step 9360/50462 | Loss: 2.7406 | LR: 4.64e-04 | 5610.4ms/step | 10.7 samples/s
|
| 705 |
+
Step 9370/50462 | Loss: 2.7582 | LR: 4.64e-04 | 5610.3ms/step | 10.7 samples/s
|
| 706 |
+
Step 9380/50462 | Loss: 2.7559 | LR: 4.64e-04 | 5607.9ms/step | 10.7 samples/s
|
| 707 |
+
Step 9390/50462 | Loss: 2.7721 | LR: 4.64e-04 | 5608.1ms/step | 10.7 samples/s
|
| 708 |
+
Step 9400/50462 | Loss: 2.7702 | LR: 4.64e-04 | 5610.2ms/step | 10.7 samples/s
|
| 709 |
+
Step 9410/50462 | Loss: 2.7491 | LR: 4.63e-04 | 5608.6ms/step | 10.7 samples/s
|
| 710 |
+
Step 9420/50462 | Loss: 2.7359 | LR: 4.63e-04 | 5605.8ms/step | 10.7 samples/s
|
| 711 |
+
Step 9430/50462 | Loss: 2.7381 | LR: 4.63e-04 | 5607.1ms/step | 10.7 samples/s
|
| 712 |
+
Step 9440/50462 | Loss: 2.7312 | LR: 4.63e-04 | 5607.7ms/step | 10.7 samples/s
|
| 713 |
+
Step 9450/50462 | Loss: 2.7743 | LR: 4.63e-04 | 5608.7ms/step | 10.7 samples/s
|
| 714 |
+
Step 9460/50462 | Loss: 2.7416 | LR: 4.63e-04 | 5607.5ms/step | 10.7 samples/s
|
| 715 |
+
Step 9470/50462 | Loss: 2.7670 | LR: 4.63e-04 | 5606.4ms/step | 10.7 samples/s
|
| 716 |
+
Step 9480/50462 | Loss: 2.7492 | LR: 4.63e-04 | 5606.1ms/step | 10.7 samples/s
|
| 717 |
+
Step 9490/50462 | Loss: 2.7678 | LR: 4.63e-04 | 5604.6ms/step | 10.7 samples/s
|
| 718 |
+
Step 9500/50462 | Loss: 2.7475 | LR: 4.63e-04 | 5606.2ms/step | 10.7 samples/s
|
| 719 |
+
Validation at step 9500: Loss = 2.8092, Perplexity = 16.60
|
| 720 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9500
|
| 721 |
+
Step 9510/50462 | Loss: 2.7561 | LR: 4.63e-04 | 8325.2ms/step | 7.2 samples/s
|
| 722 |
+
Step 9520/50462 | Loss: 2.7911 | LR: 4.63e-04 | 5604.5ms/step | 10.7 samples/s
|
| 723 |
+
Step 9530/50462 | Loss: 2.7658 | LR: 4.62e-04 | 5604.2ms/step | 10.7 samples/s
|
| 724 |
+
Step 9540/50462 | Loss: 2.7188 | LR: 4.62e-04 | 5604.3ms/step | 10.7 samples/s
|
| 725 |
+
Step 9550/50462 | Loss: 2.7615 | LR: 4.62e-04 | 5606.1ms/step | 10.7 samples/s
|
| 726 |
+
Step 9560/50462 | Loss: 2.7964 | LR: 4.62e-04 | 5605.2ms/step | 10.7 samples/s
|
| 727 |
+
Step 9570/50462 | Loss: 2.7694 | LR: 4.62e-04 | 5605.6ms/step | 10.7 samples/s
|
| 728 |
+
Step 9580/50462 | Loss: 2.7394 | LR: 4.62e-04 | 5606.6ms/step | 10.7 samples/s
|
| 729 |
+
Step 9590/50462 | Loss: 2.7903 | LR: 4.62e-04 | 5606.8ms/step | 10.7 samples/s
|
| 730 |
+
Step 9600/50462 | Loss: 2.7657 | LR: 4.62e-04 | 5604.6ms/step | 10.7 samples/s
|
| 731 |
+
Step 9610/50462 | Loss: 2.7484 | LR: 4.62e-04 | 5605.9ms/step | 10.7 samples/s
|
| 732 |
+
Step 9620/50462 | Loss: 2.7443 | LR: 4.62e-04 | 5604.5ms/step | 10.7 samples/s
|
| 733 |
+
Step 9630/50462 | Loss: 2.7518 | LR: 4.62e-04 | 5606.0ms/step | 10.7 samples/s
|
| 734 |
+
Step 9640/50462 | Loss: 2.7772 | LR: 4.62e-04 | 5605.0ms/step | 10.7 samples/s
|
| 735 |
+
Step 9650/50462 | Loss: 2.8076 | LR: 4.61e-04 | 5604.1ms/step | 10.7 samples/s
|
| 736 |
+
Step 9660/50462 | Loss: 2.7280 | LR: 4.61e-04 | 5604.8ms/step | 10.7 samples/s
|
| 737 |
+
Step 9670/50462 | Loss: 2.7233 | LR: 4.61e-04 | 5603.4ms/step | 10.7 samples/s
|
| 738 |
+
Step 9680/50462 | Loss: 2.7231 | LR: 4.61e-04 | 5604.8ms/step | 10.7 samples/s
|
| 739 |
+
Step 9690/50462 | Loss: 2.7473 | LR: 4.61e-04 | 5606.5ms/step | 10.7 samples/s
|
| 740 |
+
Step 9700/50462 | Loss: 2.7192 | LR: 4.61e-04 | 5606.6ms/step | 10.7 samples/s
|
| 741 |
+
Step 9710/50462 | Loss: 2.7704 | LR: 4.61e-04 | 5606.9ms/step | 10.7 samples/s
|
| 742 |
+
Step 9720/50462 | Loss: 2.7421 | LR: 4.61e-04 | 5604.9ms/step | 10.7 samples/s
|
| 743 |
+
Step 9730/50462 | Loss: 2.7645 | LR: 4.61e-04 | 5605.1ms/step | 10.7 samples/s
|
| 744 |
+
Step 9740/50462 | Loss: 2.7574 | LR: 4.61e-04 | 5605.9ms/step | 10.7 samples/s
|
| 745 |
+
Step 9750/50462 | Loss: 2.7395 | LR: 4.61e-04 | 5608.0ms/step | 10.7 samples/s
|
| 746 |
+
Validation at step 9750: Loss = 2.8019, Perplexity = 16.48
|
| 747 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-9750
|
| 748 |
+
Step 9760/50462 | Loss: 2.7212 | LR: 4.61e-04 | 8356.6ms/step | 7.2 samples/s
|
| 749 |
+
Step 9770/50462 | Loss: 2.7606 | LR: 4.60e-04 | 5600.8ms/step | 10.7 samples/s
|
| 750 |
+
Step 9780/50462 | Loss: 2.7806 | LR: 4.60e-04 | 5602.7ms/step | 10.7 samples/s
|
| 751 |
+
Step 9790/50462 | Loss: 2.7184 | LR: 4.60e-04 | 5603.6ms/step | 10.7 samples/s
|
| 752 |
+
Step 9800/50462 | Loss: 2.7798 | LR: 4.60e-04 | 5602.7ms/step | 10.7 samples/s
|
| 753 |
+
Step 9810/50462 | Loss: 2.7105 | LR: 4.60e-04 | 5603.7ms/step | 10.7 samples/s
|
| 754 |
+
Step 9820/50462 | Loss: 2.7196 | LR: 4.60e-04 | 5604.5ms/step | 10.7 samples/s
|
| 755 |
+
Step 9830/50462 | Loss: 2.7237 | LR: 4.60e-04 | 5605.7ms/step | 10.7 samples/s
|
| 756 |
+
Step 9840/50462 | Loss: 2.7693 | LR: 4.60e-04 | 5607.4ms/step | 10.7 samples/s
|
| 757 |
+
Step 9850/50462 | Loss: 2.7523 | LR: 4.60e-04 | 5606.8ms/step | 10.7 samples/s
|
| 758 |
+
Step 9860/50462 | Loss: 2.7584 | LR: 4.60e-04 | 5606.1ms/step | 10.7 samples/s
|
| 759 |
+
Step 9870/50462 | Loss: 2.7052 | LR: 4.60e-04 | 5606.1ms/step | 10.7 samples/s
|
| 760 |
+
Step 9880/50462 | Loss: 2.8008 | LR: 4.60e-04 | 5604.4ms/step | 10.7 samples/s
|
| 761 |
+
Step 9890/50462 | Loss: 2.7258 | LR: 4.59e-04 | 5606.0ms/step | 10.7 samples/s
|
| 762 |
+
Step 9900/50462 | Loss: 2.7258 | LR: 4.59e-04 | 5608.7ms/step | 10.7 samples/s
|
| 763 |
+
Step 9910/50462 | Loss: 2.7649 | LR: 4.59e-04 | 5606.1ms/step | 10.7 samples/s
|
| 764 |
+
Step 9920/50462 | Loss: 2.7208 | LR: 4.59e-04 | 5610.0ms/step | 10.7 samples/s
|
| 765 |
+
Step 9930/50462 | Loss: 2.7479 | LR: 4.59e-04 | 5607.0ms/step | 10.7 samples/s
|
| 766 |
+
Step 9940/50462 | Loss: 2.7224 | LR: 4.59e-04 | 5609.0ms/step | 10.7 samples/s
|
| 767 |
+
Step 9950/50462 | Loss: 2.7372 | LR: 4.59e-04 | 5606.2ms/step | 10.7 samples/s
|
| 768 |
+
Step 9960/50462 | Loss: 2.7000 | LR: 4.59e-04 | 5609.8ms/step | 10.7 samples/s
|
| 769 |
+
Step 9970/50462 | Loss: 2.7467 | LR: 4.59e-04 | 5609.7ms/step | 10.7 samples/s
|
| 770 |
+
Step 9980/50462 | Loss: 2.7468 | LR: 4.59e-04 | 5609.3ms/step | 10.7 samples/s
|
| 771 |
+
Step 9990/50462 | Loss: 2.7452 | LR: 4.59e-04 | 5609.3ms/step | 10.7 samples/s
|
| 772 |
+
Step 10000/50462 | Loss: 2.7438 | LR: 4.58e-04 | 5607.2ms/step | 10.7 samples/s
|
| 773 |
+
Validation at step 10000: Loss = 2.7972, Perplexity = 16.40
|
| 774 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10000
|
| 775 |
+
Step 10010/50462 | Loss: 2.7701 | LR: 4.58e-04 | 8397.1ms/step | 7.1 samples/s
|
| 776 |
+
Step 10020/50462 | Loss: 2.7533 | LR: 4.58e-04 | 5607.5ms/step | 10.7 samples/s
|
| 777 |
+
Step 10030/50462 | Loss: 2.7500 | LR: 4.58e-04 | 5604.4ms/step | 10.7 samples/s
|
| 778 |
+
Step 10040/50462 | Loss: 2.7883 | LR: 4.58e-04 | 5607.2ms/step | 10.7 samples/s
|
| 779 |
+
Step 10050/50462 | Loss: 2.7378 | LR: 4.58e-04 | 5602.0ms/step | 10.7 samples/s
|
| 780 |
+
Step 10060/50462 | Loss: 2.7471 | LR: 4.58e-04 | 5598.9ms/step | 10.7 samples/s
|
| 781 |
+
Step 10070/50462 | Loss: 2.7227 | LR: 4.58e-04 | 5600.5ms/step | 10.7 samples/s
|
| 782 |
+
Step 10080/50462 | Loss: 2.7259 | LR: 4.58e-04 | 5602.9ms/step | 10.7 samples/s
|
| 783 |
+
Step 10090/50462 | Loss: 2.7601 | LR: 4.58e-04 | 5601.1ms/step | 10.7 samples/s
|
| 784 |
+
Step 10100/50462 | Loss: 2.7453 | LR: 4.58e-04 | 5600.5ms/step | 10.7 samples/s
|
| 785 |
+
Step 10110/50462 | Loss: 2.7469 | LR: 4.58e-04 | 5601.9ms/step | 10.7 samples/s
|
| 786 |
+
Step 10120/50462 | Loss: 2.7661 | LR: 4.57e-04 | 5604.0ms/step | 10.7 samples/s
|
| 787 |
+
Step 10130/50462 | Loss: 2.7740 | LR: 4.57e-04 | 5600.8ms/step | 10.7 samples/s
|
| 788 |
+
Step 10140/50462 | Loss: 2.7727 | LR: 4.57e-04 | 5599.4ms/step | 10.7 samples/s
|
| 789 |
+
Step 10150/50462 | Loss: 2.7383 | LR: 4.57e-04 | 5601.9ms/step | 10.7 samples/s
|
| 790 |
+
Step 10160/50462 | Loss: 2.7311 | LR: 4.57e-04 | 5601.6ms/step | 10.7 samples/s
|
| 791 |
+
Step 10170/50462 | Loss: 2.7111 | LR: 4.57e-04 | 5601.0ms/step | 10.7 samples/s
|
| 792 |
+
Step 10180/50462 | Loss: 2.7212 | LR: 4.57e-04 | 5602.6ms/step | 10.7 samples/s
|
| 793 |
+
Step 10190/50462 | Loss: 2.7325 | LR: 4.57e-04 | 5602.4ms/step | 10.7 samples/s
|
| 794 |
+
Step 10200/50462 | Loss: 2.7312 | LR: 4.57e-04 | 5602.2ms/step | 10.7 samples/s
|
| 795 |
+
Step 10210/50462 | Loss: 2.7310 | LR: 4.57e-04 | 5604.6ms/step | 10.7 samples/s
|
| 796 |
+
Step 10220/50462 | Loss: 2.7002 | LR: 4.57e-04 | 5601.5ms/step | 10.7 samples/s
|
| 797 |
+
Step 10230/50462 | Loss: 2.7561 | LR: 4.56e-04 | 5600.3ms/step | 10.7 samples/s
|
| 798 |
+
Step 10240/50462 | Loss: 2.7453 | LR: 4.56e-04 | 5601.4ms/step | 10.7 samples/s
|
| 799 |
+
Step 10250/50462 | Loss: 2.7798 | LR: 4.56e-04 | 5600.2ms/step | 10.7 samples/s
|
| 800 |
+
Validation at step 10250: Loss = 2.7931, Perplexity = 16.33
|
| 801 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250
|
| 802 |
+
Step 10260/50462 | Loss: 2.7353 | LR: 4.56e-04 | 8450.8ms/step | 7.1 samples/s
|
| 803 |
+
Step 10270/50462 | Loss: 2.7236 | LR: 4.56e-04 | 5597.8ms/step | 10.7 samples/s
|
| 804 |
+
Step 10280/50462 | Loss: 2.7148 | LR: 4.56e-04 | 5607.9ms/step | 10.7 samples/s
|
| 805 |
+
Step 10290/50462 | Loss: 2.7074 | LR: 4.56e-04 | 5606.0ms/step | 10.7 samples/s
|
| 806 |
+
Step 10300/50462 | Loss: 2.7144 | LR: 4.56e-04 | 5608.7ms/step | 10.7 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_5gpu_relaunch.log
ADDED
|
@@ -0,0 +1,561 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 12,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 5gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
Checkpoint activations: False
|
| 74 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
================================================================================
|
| 78 |
+
MODEL ARCHITECTURE
|
| 79 |
+
================================================================================
|
| 80 |
+
Model config:
|
| 81 |
+
vocab_size: 151936
|
| 82 |
+
hidden_size: 1536
|
| 83 |
+
num_hidden_layers: 28
|
| 84 |
+
num_attention_heads: 12
|
| 85 |
+
num_key_value_heads: 4
|
| 86 |
+
head_dim: 128
|
| 87 |
+
intermediate_size: 6144
|
| 88 |
+
max_position_embeddings: 32768
|
| 89 |
+
rope_theta: 1000000
|
| 90 |
+
rope_train_position_scale_set: None
|
| 91 |
+
attention_type: sparse_stochastic
|
| 92 |
+
sparse_block_size: 128
|
| 93 |
+
sparse_num_random_blocks: 31
|
| 94 |
+
sparse_mask_pool_size: 1
|
| 95 |
+
sparse_max_seq_length: 8192
|
| 96 |
+
|
| 97 |
+
Parameter count:
|
| 98 |
+
Total parameters: 1,202,352,640
|
| 99 |
+
Trainable parameters: 1,202,352,640
|
| 100 |
+
Non-trainable parameters: 0
|
| 101 |
+
Model size (BF16): 2.40 GB
|
| 102 |
+
|
| 103 |
+
--------------------------------------------------------------------------------
|
| 104 |
+
LAYER-BY-LAYER SUMMARY
|
| 105 |
+
--------------------------------------------------------------------------------
|
| 106 |
+
================================================================================================================================================================
|
| 107 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 108 |
+
================================================================================================================================================================
|
| 109 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 110 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 111 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 112 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 113 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 142 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 143 |
+
================================================================================================================================================================
|
| 144 |
+
Total params: 1,435,726,336
|
| 145 |
+
Trainable params: 1,435,726,336
|
| 146 |
+
Non-trainable params: 0
|
| 147 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 148 |
+
================================================================================================================================================================
|
| 149 |
+
Input size (MB): 0.07
|
| 150 |
+
Forward/backward pass size (MB): 26218.59
|
| 151 |
+
Params size (MB): 2871.45
|
| 152 |
+
Estimated Total Size (MB): 29090.11
|
| 153 |
+
================================================================================================================================================================
|
| 154 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 155 |
+
================================================================================
|
| 156 |
+
|
| 157 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 158 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 162 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 163 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 164 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 165 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
Dataset split: 3027764 train, 159356 validation
|
| 170 |
+
Training examples (packed, train split): 3027764
|
| 171 |
+
Validation examples (packed, val split): 159356
|
| 172 |
+
Per-device batch size: 1
|
| 173 |
+
World size (GPUs): 5
|
| 174 |
+
Gradient accumulation steps: 12
|
| 175 |
+
Effective batch size (seqs/step): 60
|
| 176 |
+
Sequence length: 8192
|
| 177 |
+
Tokens per optimizer step: 491,520
|
| 178 |
+
Micro-batches per epoch (per rank): 605553
|
| 179 |
+
Optimizer steps per epoch: 50462
|
| 180 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 181 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10250...
|
| 182 |
+
Preserving checkpoint LR state for exact resume: current=[0.0004562829762543737], schedule_base=[0.0005]
|
| 183 |
+
Resumed from checkpoint at step 10250 (epoch 0, micro_step 137244)
|
| 184 |
+
Resuming training from step 10250
|
| 185 |
+
Resuming wandb run rgmkfpnl
|
| 186 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 187 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 188 |
+
wandb: setting up run rgmkfpnl
|
| 189 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 190 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260824_034449-rgmkfpnl
|
| 191 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 192 |
+
wandb: Resuming run 1.2b dense 8k seed70 5gpu resume
|
| 193 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 194 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 195 |
+
Starting training...
|
| 196 |
+
Validation at step 10250: Loss = 2.7931, Perplexity = 16.33
|
| 197 |
+
wandb: WARNING Tried to log to step 10250 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 198 |
+
Switching attention mode for step 10251: dense (dense step interval)
|
| 199 |
+
Step 10260/50462 | Loss: 2.7354 | LR: 4.56e-04 | 37721.5ms/step | 1.6 samples/s
|
| 200 |
+
wandb: WARNING Tried to log to step 10260 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 201 |
+
Step 10270/50462 | Loss: 2.7236 | LR: 4.56e-04 | 5615.1ms/step | 10.7 samples/s
|
| 202 |
+
wandb: WARNING Tried to log to step 10270 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 203 |
+
Step 10280/50462 | Loss: 2.7148 | LR: 4.56e-04 | 5625.1ms/step | 10.7 samples/s
|
| 204 |
+
wandb: WARNING Tried to log to step 10280 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 205 |
+
Step 10290/50462 | Loss: 2.7075 | LR: 4.56e-04 | 5629.4ms/step | 10.7 samples/s
|
| 206 |
+
wandb: WARNING Tried to log to step 10290 that is less than the current step 10291. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 207 |
+
Step 10300/50462 | Loss: 2.7144 | LR: 4.56e-04 | 5632.6ms/step | 10.7 samples/s
|
| 208 |
+
Step 10310/50462 | Loss: 2.7603 | LR: 4.56e-04 | 5633.8ms/step | 10.6 samples/s
|
| 209 |
+
Step 10320/50462 | Loss: 2.7507 | LR: 4.56e-04 | 5636.9ms/step | 10.6 samples/s
|
| 210 |
+
Step 10330/50462 | Loss: 2.7233 | LR: 4.56e-04 | 5638.8ms/step | 10.6 samples/s
|
| 211 |
+
Step 10340/50462 | Loss: 2.7692 | LR: 4.55e-04 | 5640.9ms/step | 10.6 samples/s
|
| 212 |
+
Step 10350/50462 | Loss: 2.7838 | LR: 4.55e-04 | 5641.0ms/step | 10.6 samples/s
|
| 213 |
+
Step 10360/50462 | Loss: 2.7354 | LR: 4.55e-04 | 5640.1ms/step | 10.6 samples/s
|
| 214 |
+
Step 10370/50462 | Loss: 2.7318 | LR: 4.55e-04 | 5641.2ms/step | 10.6 samples/s
|
| 215 |
+
Step 10380/50462 | Loss: 2.6996 | LR: 4.55e-04 | 5640.3ms/step | 10.6 samples/s
|
| 216 |
+
Step 10390/50462 | Loss: 2.7131 | LR: 4.55e-04 | 5641.3ms/step | 10.6 samples/s
|
| 217 |
+
Step 10400/50462 | Loss: 2.7327 | LR: 4.55e-04 | 5642.0ms/step | 10.6 samples/s
|
| 218 |
+
Step 10410/50462 | Loss: 2.7648 | LR: 4.55e-04 | 5640.8ms/step | 10.6 samples/s
|
| 219 |
+
Step 10420/50462 | Loss: 2.7247 | LR: 4.55e-04 | 5640.0ms/step | 10.6 samples/s
|
| 220 |
+
Step 10430/50462 | Loss: 2.7628 | LR: 4.55e-04 | 5641.7ms/step | 10.6 samples/s
|
| 221 |
+
Step 10440/50462 | Loss: 2.7624 | LR: 4.55e-04 | 5639.9ms/step | 10.6 samples/s
|
| 222 |
+
Step 10450/50462 | Loss: 2.7321 | LR: 4.54e-04 | 5641.0ms/step | 10.6 samples/s
|
| 223 |
+
Step 10460/50462 | Loss: 2.7171 | LR: 4.54e-04 | 5640.7ms/step | 10.6 samples/s
|
| 224 |
+
Step 10470/50462 | Loss: 2.7126 | LR: 4.54e-04 | 5641.5ms/step | 10.6 samples/s
|
| 225 |
+
Step 10480/50462 | Loss: 2.7171 | LR: 4.54e-04 | 5642.1ms/step | 10.6 samples/s
|
| 226 |
+
Step 10490/50462 | Loss: 2.7600 | LR: 4.54e-04 | 5641.4ms/step | 10.6 samples/s
|
| 227 |
+
Step 10500/50462 | Loss: 2.7462 | LR: 4.54e-04 | 5641.0ms/step | 10.6 samples/s
|
| 228 |
+
Validation at step 10500: Loss = 2.7867, Perplexity = 16.23
|
| 229 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10500
|
| 230 |
+
Step 10510/50462 | Loss: 2.7378 | LR: 4.54e-04 | 8481.1ms/step | 7.1 samples/s
|
| 231 |
+
Step 10520/50462 | Loss: 2.6880 | LR: 4.54e-04 | 5627.9ms/step | 10.7 samples/s
|
| 232 |
+
Step 10530/50462 | Loss: 2.7058 | LR: 4.54e-04 | 5633.0ms/step | 10.7 samples/s
|
| 233 |
+
Step 10540/50462 | Loss: 2.7011 | LR: 4.54e-04 | 5635.7ms/step | 10.6 samples/s
|
| 234 |
+
Step 10550/50462 | Loss: 2.7068 | LR: 4.54e-04 | 5635.9ms/step | 10.6 samples/s
|
| 235 |
+
Step 10560/50462 | Loss: 2.7737 | LR: 4.53e-04 | 5635.8ms/step | 10.6 samples/s
|
| 236 |
+
Step 10570/50462 | Loss: 2.7271 | LR: 4.53e-04 | 5637.4ms/step | 10.6 samples/s
|
| 237 |
+
Step 10580/50462 | Loss: 2.7520 | LR: 4.53e-04 | 5637.5ms/step | 10.6 samples/s
|
| 238 |
+
Step 10590/50462 | Loss: 2.7440 | LR: 4.53e-04 | 5642.9ms/step | 10.6 samples/s
|
| 239 |
+
Step 10600/50462 | Loss: 2.7655 | LR: 4.53e-04 | 5636.8ms/step | 10.6 samples/s
|
| 240 |
+
Step 10610/50462 | Loss: 2.7466 | LR: 4.53e-04 | 5643.1ms/step | 10.6 samples/s
|
| 241 |
+
Step 10620/50462 | Loss: 2.7075 | LR: 4.53e-04 | 5643.2ms/step | 10.6 samples/s
|
| 242 |
+
Step 10630/50462 | Loss: 2.7480 | LR: 4.53e-04 | 5641.5ms/step | 10.6 samples/s
|
| 243 |
+
Step 10640/50462 | Loss: 2.7565 | LR: 4.53e-04 | 5641.9ms/step | 10.6 samples/s
|
| 244 |
+
Step 10650/50462 | Loss: 2.7419 | LR: 4.53e-04 | 5640.4ms/step | 10.6 samples/s
|
| 245 |
+
Step 10660/50462 | Loss: 2.7346 | LR: 4.53e-04 | 5643.2ms/step | 10.6 samples/s
|
| 246 |
+
Step 10670/50462 | Loss: 2.7581 | LR: 4.52e-04 | 5638.9ms/step | 10.6 samples/s
|
| 247 |
+
Step 10680/50462 | Loss: 2.7524 | LR: 4.52e-04 | 5640.1ms/step | 10.6 samples/s
|
| 248 |
+
Step 10690/50462 | Loss: 2.7355 | LR: 4.52e-04 | 5640.8ms/step | 10.6 samples/s
|
| 249 |
+
Step 10700/50462 | Loss: 2.7895 | LR: 4.52e-04 | 5641.6ms/step | 10.6 samples/s
|
| 250 |
+
Step 10710/50462 | Loss: 2.7083 | LR: 4.52e-04 | 5641.3ms/step | 10.6 samples/s
|
| 251 |
+
Step 10720/50462 | Loss: 2.7228 | LR: 4.52e-04 | 5641.2ms/step | 10.6 samples/s
|
| 252 |
+
Step 10730/50462 | Loss: 2.7448 | LR: 4.52e-04 | 5639.3ms/step | 10.6 samples/s
|
| 253 |
+
Step 10740/50462 | Loss: 2.7277 | LR: 4.52e-04 | 5640.0ms/step | 10.6 samples/s
|
| 254 |
+
Step 10750/50462 | Loss: 2.7237 | LR: 4.52e-04 | 5641.2ms/step | 10.6 samples/s
|
| 255 |
+
Validation at step 10750: Loss = 2.7808, Perplexity = 16.13
|
| 256 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-10750
|
| 257 |
+
Step 10760/50462 | Loss: 2.7602 | LR: 4.52e-04 | 8396.0ms/step | 7.1 samples/s
|
| 258 |
+
Step 10770/50462 | Loss: 2.7259 | LR: 4.52e-04 | 5637.2ms/step | 10.6 samples/s
|
| 259 |
+
Step 10780/50462 | Loss: 2.7359 | LR: 4.51e-04 | 5644.2ms/step | 10.6 samples/s
|
| 260 |
+
Step 10790/50462 | Loss: 2.7434 | LR: 4.51e-04 | 5645.6ms/step | 10.6 samples/s
|
| 261 |
+
Step 10800/50462 | Loss: 2.6968 | LR: 4.51e-04 | 5644.3ms/step | 10.6 samples/s
|
| 262 |
+
Step 10810/50462 | Loss: 2.7757 | LR: 4.51e-04 | 5645.7ms/step | 10.6 samples/s
|
| 263 |
+
Step 10820/50462 | Loss: 2.7331 | LR: 4.51e-04 | 5642.8ms/step | 10.6 samples/s
|
| 264 |
+
Step 10830/50462 | Loss: 2.7447 | LR: 4.51e-04 | 5646.1ms/step | 10.6 samples/s
|
| 265 |
+
Step 10840/50462 | Loss: 2.7243 | LR: 4.51e-04 | 5648.7ms/step | 10.6 samples/s
|
| 266 |
+
Step 10850/50462 | Loss: 2.7122 | LR: 4.51e-04 | 5647.4ms/step | 10.6 samples/s
|
| 267 |
+
Step 10860/50462 | Loss: 2.7296 | LR: 4.51e-04 | 5647.2ms/step | 10.6 samples/s
|
| 268 |
+
Step 10870/50462 | Loss: 2.7203 | LR: 4.51e-04 | 5647.8ms/step | 10.6 samples/s
|
| 269 |
+
Step 10880/50462 | Loss: 2.7106 | LR: 4.50e-04 | 5646.7ms/step | 10.6 samples/s
|
| 270 |
+
Step 10890/50462 | Loss: 2.7227 | LR: 4.50e-04 | 5645.7ms/step | 10.6 samples/s
|
| 271 |
+
Step 10900/50462 | Loss: 2.7020 | LR: 4.50e-04 | 5647.6ms/step | 10.6 samples/s
|
| 272 |
+
Step 10910/50462 | Loss: 2.6824 | LR: 4.50e-04 | 5647.9ms/step | 10.6 samples/s
|
| 273 |
+
Step 10920/50462 | Loss: 2.7373 | LR: 4.50e-04 | 5647.2ms/step | 10.6 samples/s
|
| 274 |
+
Step 10930/50462 | Loss: 2.6812 | LR: 4.50e-04 | 5647.2ms/step | 10.6 samples/s
|
| 275 |
+
Step 10940/50462 | Loss: 2.7121 | LR: 4.50e-04 | 5645.4ms/step | 10.6 samples/s
|
| 276 |
+
Step 10950/50462 | Loss: 2.7512 | LR: 4.50e-04 | 5644.8ms/step | 10.6 samples/s
|
| 277 |
+
Step 10960/50462 | Loss: 2.6994 | LR: 4.50e-04 | 5645.8ms/step | 10.6 samples/s
|
| 278 |
+
Step 10970/50462 | Loss: 2.7474 | LR: 4.50e-04 | 5646.3ms/step | 10.6 samples/s
|
| 279 |
+
Step 10980/50462 | Loss: 2.7477 | LR: 4.50e-04 | 5645.8ms/step | 10.6 samples/s
|
| 280 |
+
Step 10990/50462 | Loss: 2.7079 | LR: 4.49e-04 | 5645.8ms/step | 10.6 samples/s
|
| 281 |
+
Step 11000/50462 | Loss: 2.7096 | LR: 4.49e-04 | 5647.5ms/step | 10.6 samples/s
|
| 282 |
+
Validation at step 11000: Loss = 2.7744, Perplexity = 16.03
|
| 283 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11000
|
| 284 |
+
Step 11010/50462 | Loss: 2.7294 | LR: 4.49e-04 | 8370.3ms/step | 7.2 samples/s
|
| 285 |
+
Step 11020/50462 | Loss: 2.7313 | LR: 4.49e-04 | 5642.0ms/step | 10.6 samples/s
|
| 286 |
+
Step 11030/50462 | Loss: 2.7250 | LR: 4.49e-04 | 5642.9ms/step | 10.6 samples/s
|
| 287 |
+
Step 11040/50462 | Loss: 2.7164 | LR: 4.49e-04 | 5644.1ms/step | 10.6 samples/s
|
| 288 |
+
Step 11050/50462 | Loss: 2.7280 | LR: 4.49e-04 | 5643.8ms/step | 10.6 samples/s
|
| 289 |
+
Step 11060/50462 | Loss: 2.7219 | LR: 4.49e-04 | 5643.6ms/step | 10.6 samples/s
|
| 290 |
+
Step 11070/50462 | Loss: 2.7605 | LR: 4.49e-04 | 5643.7ms/step | 10.6 samples/s
|
| 291 |
+
Step 11080/50462 | Loss: 2.6966 | LR: 4.49e-04 | 5645.3ms/step | 10.6 samples/s
|
| 292 |
+
Step 11090/50462 | Loss: 2.7335 | LR: 4.48e-04 | 5645.1ms/step | 10.6 samples/s
|
| 293 |
+
Step 11100/50462 | Loss: 2.7150 | LR: 4.48e-04 | 5644.6ms/step | 10.6 samples/s
|
| 294 |
+
Step 11110/50462 | Loss: 2.7271 | LR: 4.48e-04 | 5647.0ms/step | 10.6 samples/s
|
| 295 |
+
Step 11120/50462 | Loss: 2.7073 | LR: 4.48e-04 | 5644.3ms/step | 10.6 samples/s
|
| 296 |
+
Step 11130/50462 | Loss: 2.7497 | LR: 4.48e-04 | 5644.7ms/step | 10.6 samples/s
|
| 297 |
+
Step 11140/50462 | Loss: 2.7122 | LR: 4.48e-04 | 5644.3ms/step | 10.6 samples/s
|
| 298 |
+
Step 11150/50462 | Loss: 2.7437 | LR: 4.48e-04 | 5645.6ms/step | 10.6 samples/s
|
| 299 |
+
Step 11160/50462 | Loss: 2.7316 | LR: 4.48e-04 | 5645.0ms/step | 10.6 samples/s
|
| 300 |
+
Step 11170/50462 | Loss: 2.7044 | LR: 4.48e-04 | 5644.8ms/step | 10.6 samples/s
|
| 301 |
+
Step 11180/50462 | Loss: 2.7250 | LR: 4.48e-04 | 5644.5ms/step | 10.6 samples/s
|
| 302 |
+
Step 11190/50462 | Loss: 2.7188 | LR: 4.48e-04 | 5646.4ms/step | 10.6 samples/s
|
| 303 |
+
Step 11200/50462 | Loss: 2.7258 | LR: 4.47e-04 | 5645.4ms/step | 10.6 samples/s
|
| 304 |
+
Step 11210/50462 | Loss: 2.6904 | LR: 4.47e-04 | 5644.7ms/step | 10.6 samples/s
|
| 305 |
+
Step 11220/50462 | Loss: 2.7061 | LR: 4.47e-04 | 5645.8ms/step | 10.6 samples/s
|
| 306 |
+
Step 11230/50462 | Loss: 2.7366 | LR: 4.47e-04 | 5646.0ms/step | 10.6 samples/s
|
| 307 |
+
Step 11240/50462 | Loss: 2.7077 | LR: 4.47e-04 | 5644.8ms/step | 10.6 samples/s
|
| 308 |
+
Step 11250/50462 | Loss: 2.7370 | LR: 4.47e-04 | 5645.0ms/step | 10.6 samples/s
|
| 309 |
+
Validation at step 11250: Loss = 2.7708, Perplexity = 15.97
|
| 310 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11250
|
| 311 |
+
Step 11260/50462 | Loss: 2.7051 | LR: 4.47e-04 | 8360.7ms/step | 7.2 samples/s
|
| 312 |
+
Step 11270/50462 | Loss: 2.7480 | LR: 4.47e-04 | 5641.4ms/step | 10.6 samples/s
|
| 313 |
+
Step 11280/50462 | Loss: 2.7145 | LR: 4.47e-04 | 5649.5ms/step | 10.6 samples/s
|
| 314 |
+
Step 11290/50462 | Loss: 2.7190 | LR: 4.47e-04 | 5650.9ms/step | 10.6 samples/s
|
| 315 |
+
Step 11300/50462 | Loss: 2.7363 | LR: 4.46e-04 | 5651.7ms/step | 10.6 samples/s
|
| 316 |
+
Step 11310/50462 | Loss: 2.7248 | LR: 4.46e-04 | 5651.0ms/step | 10.6 samples/s
|
| 317 |
+
Step 11320/50462 | Loss: 2.6909 | LR: 4.46e-04 | 5651.5ms/step | 10.6 samples/s
|
| 318 |
+
Step 11330/50462 | Loss: 2.7174 | LR: 4.46e-04 | 5653.4ms/step | 10.6 samples/s
|
| 319 |
+
Step 11340/50462 | Loss: 2.7418 | LR: 4.46e-04 | 5651.7ms/step | 10.6 samples/s
|
| 320 |
+
Step 11350/50462 | Loss: 2.7546 | LR: 4.46e-04 | 5651.5ms/step | 10.6 samples/s
|
| 321 |
+
Step 11360/50462 | Loss: 2.7314 | LR: 4.46e-04 | 5653.3ms/step | 10.6 samples/s
|
| 322 |
+
Step 11370/50462 | Loss: 2.7261 | LR: 4.46e-04 | 5652.7ms/step | 10.6 samples/s
|
| 323 |
+
Step 11380/50462 | Loss: 2.7549 | LR: 4.46e-04 | 5650.7ms/step | 10.6 samples/s
|
| 324 |
+
Step 11390/50462 | Loss: 2.7228 | LR: 4.46e-04 | 5652.2ms/step | 10.6 samples/s
|
| 325 |
+
Step 11400/50462 | Loss: 2.6983 | LR: 4.45e-04 | 5652.0ms/step | 10.6 samples/s
|
| 326 |
+
Step 11410/50462 | Loss: 2.7173 | LR: 4.45e-04 | 5651.5ms/step | 10.6 samples/s
|
| 327 |
+
Step 11420/50462 | Loss: 2.7709 | LR: 4.45e-04 | 5651.8ms/step | 10.6 samples/s
|
| 328 |
+
Step 11430/50462 | Loss: 2.6770 | LR: 4.45e-04 | 5653.2ms/step | 10.6 samples/s
|
| 329 |
+
Step 11440/50462 | Loss: 2.6962 | LR: 4.45e-04 | 5651.9ms/step | 10.6 samples/s
|
| 330 |
+
Step 11450/50462 | Loss: 2.6749 | LR: 4.45e-04 | 5651.1ms/step | 10.6 samples/s
|
| 331 |
+
Step 11460/50462 | Loss: 2.7105 | LR: 4.45e-04 | 5652.0ms/step | 10.6 samples/s
|
| 332 |
+
Step 11470/50462 | Loss: 2.7485 | LR: 4.45e-04 | 5650.9ms/step | 10.6 samples/s
|
| 333 |
+
Step 11480/50462 | Loss: 2.7518 | LR: 4.45e-04 | 5653.7ms/step | 10.6 samples/s
|
| 334 |
+
Step 11490/50462 | Loss: 2.6540 | LR: 4.45e-04 | 5658.7ms/step | 10.6 samples/s
|
| 335 |
+
Step 11500/50462 | Loss: 2.7453 | LR: 4.44e-04 | 5661.7ms/step | 10.6 samples/s
|
| 336 |
+
Validation at step 11500: Loss = 2.7652, Perplexity = 15.88
|
| 337 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11500
|
| 338 |
+
Step 11510/50462 | Loss: 2.6906 | LR: 4.44e-04 | 8349.0ms/step | 7.2 samples/s
|
| 339 |
+
Step 11520/50462 | Loss: 2.6969 | LR: 4.44e-04 | 5640.6ms/step | 10.6 samples/s
|
| 340 |
+
Step 11530/50462 | Loss: 2.6865 | LR: 4.44e-04 | 5642.2ms/step | 10.6 samples/s
|
| 341 |
+
Step 11540/50462 | Loss: 2.7038 | LR: 4.44e-04 | 5641.9ms/step | 10.6 samples/s
|
| 342 |
+
Step 11550/50462 | Loss: 2.7799 | LR: 4.44e-04 | 5642.0ms/step | 10.6 samples/s
|
| 343 |
+
Step 11560/50462 | Loss: 2.7034 | LR: 4.44e-04 | 5642.2ms/step | 10.6 samples/s
|
| 344 |
+
Step 11570/50462 | Loss: 2.7148 | LR: 4.44e-04 | 5642.7ms/step | 10.6 samples/s
|
| 345 |
+
Step 11580/50462 | Loss: 2.7276 | LR: 4.44e-04 | 5643.5ms/step | 10.6 samples/s
|
| 346 |
+
Step 11590/50462 | Loss: 2.7168 | LR: 4.44e-04 | 5644.1ms/step | 10.6 samples/s
|
| 347 |
+
Step 11600/50462 | Loss: 2.7288 | LR: 4.43e-04 | 5644.8ms/step | 10.6 samples/s
|
| 348 |
+
Step 11610/50462 | Loss: 2.7339 | LR: 4.43e-04 | 5645.5ms/step | 10.6 samples/s
|
| 349 |
+
Step 11620/50462 | Loss: 2.6936 | LR: 4.43e-04 | 5644.0ms/step | 10.6 samples/s
|
| 350 |
+
Step 11630/50462 | Loss: 2.7077 | LR: 4.43e-04 | 5643.1ms/step | 10.6 samples/s
|
| 351 |
+
Step 11640/50462 | Loss: 2.7321 | LR: 4.43e-04 | 5642.4ms/step | 10.6 samples/s
|
| 352 |
+
Step 11650/50462 | Loss: 2.7254 | LR: 4.43e-04 | 5640.5ms/step | 10.6 samples/s
|
| 353 |
+
Step 11660/50462 | Loss: 2.7044 | LR: 4.43e-04 | 5643.9ms/step | 10.6 samples/s
|
| 354 |
+
Step 11670/50462 | Loss: 2.7206 | LR: 4.43e-04 | 5642.4ms/step | 10.6 samples/s
|
| 355 |
+
Step 11680/50462 | Loss: 2.6768 | LR: 4.43e-04 | 5643.5ms/step | 10.6 samples/s
|
| 356 |
+
Step 11690/50462 | Loss: 2.6504 | LR: 4.43e-04 | 5644.8ms/step | 10.6 samples/s
|
| 357 |
+
Step 11700/50462 | Loss: 2.7350 | LR: 4.42e-04 | 5644.3ms/step | 10.6 samples/s
|
| 358 |
+
Step 11710/50462 | Loss: 2.6976 | LR: 4.42e-04 | 5643.4ms/step | 10.6 samples/s
|
| 359 |
+
Step 11720/50462 | Loss: 2.6999 | LR: 4.42e-04 | 5643.1ms/step | 10.6 samples/s
|
| 360 |
+
Step 11730/50462 | Loss: 2.6743 | LR: 4.42e-04 | 5639.8ms/step | 10.6 samples/s
|
| 361 |
+
Step 11740/50462 | Loss: 2.6739 | LR: 4.42e-04 | 5650.4ms/step | 10.6 samples/s
|
| 362 |
+
Step 11750/50462 | Loss: 2.6973 | LR: 4.42e-04 | 5647.1ms/step | 10.6 samples/s
|
| 363 |
+
Validation at step 11750: Loss = 2.7606, Perplexity = 15.81
|
| 364 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-11750
|
| 365 |
+
Step 11760/50462 | Loss: 2.6288 | LR: 4.42e-04 | 8366.0ms/step | 7.2 samples/s
|
| 366 |
+
Step 11770/50462 | Loss: 2.7083 | LR: 4.42e-04 | 5632.9ms/step | 10.7 samples/s
|
| 367 |
+
Step 11780/50462 | Loss: 2.7527 | LR: 4.42e-04 | 5635.5ms/step | 10.6 samples/s
|
| 368 |
+
Step 11790/50462 | Loss: 2.6870 | LR: 4.42e-04 | 5637.8ms/step | 10.6 samples/s
|
| 369 |
+
Step 11800/50462 | Loss: 2.6897 | LR: 4.41e-04 | 5636.1ms/step | 10.6 samples/s
|
| 370 |
+
Step 11810/50462 | Loss: 2.7512 | LR: 4.41e-04 | 5638.0ms/step | 10.6 samples/s
|
| 371 |
+
Step 11820/50462 | Loss: 2.7440 | LR: 4.41e-04 | 5637.2ms/step | 10.6 samples/s
|
| 372 |
+
Step 11830/50462 | Loss: 2.7021 | LR: 4.41e-04 | 5638.2ms/step | 10.6 samples/s
|
| 373 |
+
Step 11840/50462 | Loss: 2.6978 | LR: 4.41e-04 | 5637.4ms/step | 10.6 samples/s
|
| 374 |
+
Step 11850/50462 | Loss: 2.7295 | LR: 4.41e-04 | 5637.6ms/step | 10.6 samples/s
|
| 375 |
+
Step 11860/50462 | Loss: 2.6906 | LR: 4.41e-04 | 5636.7ms/step | 10.6 samples/s
|
| 376 |
+
Step 11870/50462 | Loss: 2.6766 | LR: 4.41e-04 | 5635.8ms/step | 10.6 samples/s
|
| 377 |
+
Step 11880/50462 | Loss: 2.7231 | LR: 4.41e-04 | 5637.7ms/step | 10.6 samples/s
|
| 378 |
+
Step 11890/50462 | Loss: 2.7020 | LR: 4.41e-04 | 5636.8ms/step | 10.6 samples/s
|
| 379 |
+
Step 11900/50462 | Loss: 2.7333 | LR: 4.40e-04 | 5639.0ms/step | 10.6 samples/s
|
| 380 |
+
Step 11910/50462 | Loss: 2.7219 | LR: 4.40e-04 | 5641.7ms/step | 10.6 samples/s
|
| 381 |
+
Step 11920/50462 | Loss: 2.7254 | LR: 4.40e-04 | 5640.0ms/step | 10.6 samples/s
|
| 382 |
+
Step 11930/50462 | Loss: 2.6999 | LR: 4.40e-04 | 5638.8ms/step | 10.6 samples/s
|
| 383 |
+
Step 11940/50462 | Loss: 2.6967 | LR: 4.40e-04 | 5639.3ms/step | 10.6 samples/s
|
| 384 |
+
Step 11950/50462 | Loss: 2.6480 | LR: 4.40e-04 | 5637.0ms/step | 10.6 samples/s
|
| 385 |
+
Step 11960/50462 | Loss: 2.7282 | LR: 4.40e-04 | 5638.3ms/step | 10.6 samples/s
|
| 386 |
+
Step 11970/50462 | Loss: 2.7551 | LR: 4.40e-04 | 5638.1ms/step | 10.6 samples/s
|
| 387 |
+
Step 11980/50462 | Loss: 2.7355 | LR: 4.40e-04 | 5638.3ms/step | 10.6 samples/s
|
| 388 |
+
Step 11990/50462 | Loss: 2.6881 | LR: 4.40e-04 | 5638.4ms/step | 10.6 samples/s
|
| 389 |
+
Step 12000/50462 | Loss: 2.6719 | LR: 4.39e-04 | 5639.1ms/step | 10.6 samples/s
|
| 390 |
+
Validation at step 12000: Loss = 2.7572, Perplexity = 15.76
|
| 391 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12000
|
| 392 |
+
Step 12010/50462 | Loss: 2.7396 | LR: 4.39e-04 | 8379.4ms/step | 7.2 samples/s
|
| 393 |
+
Step 12020/50462 | Loss: 2.7468 | LR: 4.39e-04 | 5626.9ms/step | 10.7 samples/s
|
| 394 |
+
Step 12030/50462 | Loss: 2.6915 | LR: 4.39e-04 | 5633.8ms/step | 10.7 samples/s
|
| 395 |
+
Step 12040/50462 | Loss: 2.6746 | LR: 4.39e-04 | 5637.0ms/step | 10.6 samples/s
|
| 396 |
+
Step 12050/50462 | Loss: 2.6716 | LR: 4.39e-04 | 5640.0ms/step | 10.6 samples/s
|
| 397 |
+
Step 12060/50462 | Loss: 2.7175 | LR: 4.39e-04 | 5638.6ms/step | 10.6 samples/s
|
| 398 |
+
Step 12070/50462 | Loss: 2.6792 | LR: 4.39e-04 | 5639.1ms/step | 10.6 samples/s
|
| 399 |
+
Step 12080/50462 | Loss: 2.7227 | LR: 4.39e-04 | 5639.2ms/step | 10.6 samples/s
|
| 400 |
+
Step 12090/50462 | Loss: 2.7076 | LR: 4.39e-04 | 5640.5ms/step | 10.6 samples/s
|
| 401 |
+
Step 12100/50462 | Loss: 2.6879 | LR: 4.38e-04 | 5638.8ms/step | 10.6 samples/s
|
| 402 |
+
Step 12110/50462 | Loss: 2.7421 | LR: 4.38e-04 | 5638.7ms/step | 10.6 samples/s
|
| 403 |
+
Step 12120/50462 | Loss: 2.7086 | LR: 4.38e-04 | 5639.5ms/step | 10.6 samples/s
|
| 404 |
+
Step 12130/50462 | Loss: 2.7030 | LR: 4.38e-04 | 5638.9ms/step | 10.6 samples/s
|
| 405 |
+
Step 12140/50462 | Loss: 2.7004 | LR: 4.38e-04 | 5640.9ms/step | 10.6 samples/s
|
| 406 |
+
Step 12150/50462 | Loss: 2.7036 | LR: 4.38e-04 | 5640.5ms/step | 10.6 samples/s
|
| 407 |
+
Step 12160/50462 | Loss: 2.7185 | LR: 4.38e-04 | 5638.6ms/step | 10.6 samples/s
|
| 408 |
+
Step 12170/50462 | Loss: 2.7159 | LR: 4.38e-04 | 5638.6ms/step | 10.6 samples/s
|
| 409 |
+
Step 12180/50462 | Loss: 2.7379 | LR: 4.38e-04 | 5638.2ms/step | 10.6 samples/s
|
| 410 |
+
Step 12190/50462 | Loss: 2.7616 | LR: 4.37e-04 | 5637.4ms/step | 10.6 samples/s
|
| 411 |
+
Step 12200/50462 | Loss: 2.6951 | LR: 4.37e-04 | 5637.9ms/step | 10.6 samples/s
|
| 412 |
+
Step 12210/50462 | Loss: 2.6838 | LR: 4.37e-04 | 5638.2ms/step | 10.6 samples/s
|
| 413 |
+
Step 12220/50462 | Loss: 2.6739 | LR: 4.37e-04 | 5639.0ms/step | 10.6 samples/s
|
| 414 |
+
Step 12230/50462 | Loss: 2.7626 | LR: 4.37e-04 | 5640.1ms/step | 10.6 samples/s
|
| 415 |
+
Step 12240/50462 | Loss: 2.7011 | LR: 4.37e-04 | 5639.5ms/step | 10.6 samples/s
|
| 416 |
+
Step 12250/50462 | Loss: 2.6758 | LR: 4.37e-04 | 5639.7ms/step | 10.6 samples/s
|
| 417 |
+
Validation at step 12250: Loss = 2.7546, Perplexity = 15.72
|
| 418 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12250
|
| 419 |
+
Step 12260/50462 | Loss: 2.6675 | LR: 4.37e-04 | 8383.5ms/step | 7.2 samples/s
|
| 420 |
+
Step 12270/50462 | Loss: 2.7184 | LR: 4.37e-04 | 5629.0ms/step | 10.7 samples/s
|
| 421 |
+
Step 12280/50462 | Loss: 2.6815 | LR: 4.37e-04 | 5635.1ms/step | 10.6 samples/s
|
| 422 |
+
Step 12290/50462 | Loss: 2.7288 | LR: 4.36e-04 | 5638.3ms/step | 10.6 samples/s
|
| 423 |
+
Step 12300/50462 | Loss: 2.7355 | LR: 4.36e-04 | 5637.1ms/step | 10.6 samples/s
|
| 424 |
+
Step 12310/50462 | Loss: 2.7291 | LR: 4.36e-04 | 5638.1ms/step | 10.6 samples/s
|
| 425 |
+
Step 12320/50462 | Loss: 2.6975 | LR: 4.36e-04 | 5637.8ms/step | 10.6 samples/s
|
| 426 |
+
Step 12330/50462 | Loss: 2.7134 | LR: 4.36e-04 | 5637.4ms/step | 10.6 samples/s
|
| 427 |
+
Step 12340/50462 | Loss: 2.6923 | LR: 4.36e-04 | 5637.9ms/step | 10.6 samples/s
|
| 428 |
+
Step 12350/50462 | Loss: 2.7155 | LR: 4.36e-04 | 5636.6ms/step | 10.6 samples/s
|
| 429 |
+
Step 12360/50462 | Loss: 2.7111 | LR: 4.36e-04 | 5639.0ms/step | 10.6 samples/s
|
| 430 |
+
Step 12370/50462 | Loss: 2.7175 | LR: 4.36e-04 | 5636.5ms/step | 10.6 samples/s
|
| 431 |
+
Step 12380/50462 | Loss: 2.7681 | LR: 4.35e-04 | 5639.3ms/step | 10.6 samples/s
|
| 432 |
+
Step 12390/50462 | Loss: 2.7098 | LR: 4.35e-04 | 5638.5ms/step | 10.6 samples/s
|
| 433 |
+
Step 12400/50462 | Loss: 2.6669 | LR: 4.35e-04 | 5639.2ms/step | 10.6 samples/s
|
| 434 |
+
Step 12410/50462 | Loss: 2.7098 | LR: 4.35e-04 | 5638.9ms/step | 10.6 samples/s
|
| 435 |
+
Step 12420/50462 | Loss: 2.7075 | LR: 4.35e-04 | 5639.7ms/step | 10.6 samples/s
|
| 436 |
+
Step 12430/50462 | Loss: 2.6992 | LR: 4.35e-04 | 5639.2ms/step | 10.6 samples/s
|
| 437 |
+
Step 12440/50462 | Loss: 2.7315 | LR: 4.35e-04 | 5638.8ms/step | 10.6 samples/s
|
| 438 |
+
Step 12450/50462 | Loss: 2.6956 | LR: 4.35e-04 | 5639.5ms/step | 10.6 samples/s
|
| 439 |
+
Step 12460/50462 | Loss: 2.7096 | LR: 4.35e-04 | 5640.2ms/step | 10.6 samples/s
|
| 440 |
+
Step 12470/50462 | Loss: 2.7177 | LR: 4.35e-04 | 5639.1ms/step | 10.6 samples/s
|
| 441 |
+
Step 12480/50462 | Loss: 2.7254 | LR: 4.34e-04 | 5639.4ms/step | 10.6 samples/s
|
| 442 |
+
Step 12490/50462 | Loss: 2.7152 | LR: 4.34e-04 | 5638.3ms/step | 10.6 samples/s
|
| 443 |
+
Step 12500/50462 | Loss: 2.7136 | LR: 4.34e-04 | 5637.0ms/step | 10.6 samples/s
|
| 444 |
+
Validation at step 12500: Loss = 2.7509, Perplexity = 15.66
|
| 445 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12500
|
| 446 |
+
Step 12510/50462 | Loss: 2.6697 | LR: 4.34e-04 | 8368.0ms/step | 7.2 samples/s
|
| 447 |
+
Step 12520/50462 | Loss: 2.7041 | LR: 4.34e-04 | 5630.3ms/step | 10.7 samples/s
|
| 448 |
+
Step 12530/50462 | Loss: 2.7109 | LR: 4.34e-04 | 5636.5ms/step | 10.6 samples/s
|
| 449 |
+
Step 12540/50462 | Loss: 2.6784 | LR: 4.34e-04 | 5637.5ms/step | 10.6 samples/s
|
| 450 |
+
Step 12550/50462 | Loss: 2.7024 | LR: 4.34e-04 | 5638.8ms/step | 10.6 samples/s
|
| 451 |
+
Step 12560/50462 | Loss: 2.6931 | LR: 4.34e-04 | 5638.8ms/step | 10.6 samples/s
|
| 452 |
+
Step 12570/50462 | Loss: 2.6884 | LR: 4.33e-04 | 5637.4ms/step | 10.6 samples/s
|
| 453 |
+
Step 12580/50462 | Loss: 2.6848 | LR: 4.33e-04 | 5640.3ms/step | 10.6 samples/s
|
| 454 |
+
Step 12590/50462 | Loss: 2.7059 | LR: 4.33e-04 | 5639.3ms/step | 10.6 samples/s
|
| 455 |
+
Step 12600/50462 | Loss: 2.7072 | LR: 4.33e-04 | 5638.7ms/step | 10.6 samples/s
|
| 456 |
+
Step 12610/50462 | Loss: 2.7055 | LR: 4.33e-04 | 5640.0ms/step | 10.6 samples/s
|
| 457 |
+
Step 12620/50462 | Loss: 2.6985 | LR: 4.33e-04 | 5639.7ms/step | 10.6 samples/s
|
| 458 |
+
Step 12630/50462 | Loss: 2.7075 | LR: 4.33e-04 | 5637.0ms/step | 10.6 samples/s
|
| 459 |
+
Step 12640/50462 | Loss: 2.7099 | LR: 4.33e-04 | 5637.9ms/step | 10.6 samples/s
|
| 460 |
+
Step 12650/50462 | Loss: 2.7097 | LR: 4.33e-04 | 5638.6ms/step | 10.6 samples/s
|
| 461 |
+
Step 12660/50462 | Loss: 2.6667 | LR: 4.32e-04 | 5638.6ms/step | 10.6 samples/s
|
| 462 |
+
Step 12670/50462 | Loss: 2.7050 | LR: 4.32e-04 | 5639.0ms/step | 10.6 samples/s
|
| 463 |
+
Step 12680/50462 | Loss: 2.6851 | LR: 4.32e-04 | 5639.9ms/step | 10.6 samples/s
|
| 464 |
+
Step 12690/50462 | Loss: 2.6891 | LR: 4.32e-04 | 5640.2ms/step | 10.6 samples/s
|
| 465 |
+
Step 12700/50462 | Loss: 2.6817 | LR: 4.32e-04 | 5639.8ms/step | 10.6 samples/s
|
| 466 |
+
Step 12710/50462 | Loss: 2.6736 | LR: 4.32e-04 | 5641.0ms/step | 10.6 samples/s
|
| 467 |
+
Step 12720/50462 | Loss: 2.6906 | LR: 4.32e-04 | 5639.2ms/step | 10.6 samples/s
|
| 468 |
+
Step 12730/50462 | Loss: 2.7160 | LR: 4.32e-04 | 5640.1ms/step | 10.6 samples/s
|
| 469 |
+
Step 12740/50462 | Loss: 2.6925 | LR: 4.32e-04 | 5639.7ms/step | 10.6 samples/s
|
| 470 |
+
Step 12750/50462 | Loss: 2.6902 | LR: 4.31e-04 | 5640.5ms/step | 10.6 samples/s
|
| 471 |
+
Validation at step 12750: Loss = 2.7480, Perplexity = 15.61
|
| 472 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-12750
|
| 473 |
+
Step 12760/50462 | Loss: 2.7178 | LR: 4.31e-04 | 8366.6ms/step | 7.2 samples/s
|
| 474 |
+
Step 12770/50462 | Loss: 2.7105 | LR: 4.31e-04 | 5626.2ms/step | 10.7 samples/s
|
| 475 |
+
Step 12780/50462 | Loss: 2.6892 | LR: 4.31e-04 | 5634.1ms/step | 10.6 samples/s
|
| 476 |
+
Step 12790/50462 | Loss: 2.6927 | LR: 4.31e-04 | 5649.6ms/step | 10.6 samples/s
|
| 477 |
+
Step 12800/50462 | Loss: 2.7041 | LR: 4.31e-04 | 5647.6ms/step | 10.6 samples/s
|
| 478 |
+
Step 12810/50462 | Loss: 2.7128 | LR: 4.31e-04 | 5646.8ms/step | 10.6 samples/s
|
| 479 |
+
Step 12820/50462 | Loss: 2.7024 | LR: 4.31e-04 | 5647.2ms/step | 10.6 samples/s
|
| 480 |
+
Step 12830/50462 | Loss: 2.6945 | LR: 4.31e-04 | 5649.1ms/step | 10.6 samples/s
|
| 481 |
+
Step 12840/50462 | Loss: 2.6898 | LR: 4.31e-04 | 5646.9ms/step | 10.6 samples/s
|
| 482 |
+
Step 12850/50462 | Loss: 2.7150 | LR: 4.30e-04 | 5648.3ms/step | 10.6 samples/s
|
| 483 |
+
Step 12860/50462 | Loss: 2.7605 | LR: 4.30e-04 | 5644.8ms/step | 10.6 samples/s
|
| 484 |
+
Step 12870/50462 | Loss: 2.6795 | LR: 4.30e-04 | 5645.9ms/step | 10.6 samples/s
|
| 485 |
+
Step 12880/50462 | Loss: 2.7200 | LR: 4.30e-04 | 5647.0ms/step | 10.6 samples/s
|
| 486 |
+
Step 12890/50462 | Loss: 2.7309 | LR: 4.30e-04 | 5635.5ms/step | 10.6 samples/s
|
| 487 |
+
Step 12900/50462 | Loss: 2.6920 | LR: 4.30e-04 | 5635.8ms/step | 10.6 samples/s
|
| 488 |
+
Step 12910/50462 | Loss: 2.6885 | LR: 4.30e-04 | 5642.0ms/step | 10.6 samples/s
|
| 489 |
+
Step 12920/50462 | Loss: 2.7058 | LR: 4.30e-04 | 5645.4ms/step | 10.6 samples/s
|
| 490 |
+
Step 12930/50462 | Loss: 2.7011 | LR: 4.30e-04 | 5646.1ms/step | 10.6 samples/s
|
| 491 |
+
Step 12940/50462 | Loss: 2.7012 | LR: 4.29e-04 | 5646.6ms/step | 10.6 samples/s
|
| 492 |
+
Step 12950/50462 | Loss: 2.6862 | LR: 4.29e-04 | 5641.7ms/step | 10.6 samples/s
|
| 493 |
+
Step 12960/50462 | Loss: 2.6712 | LR: 4.29e-04 | 5640.8ms/step | 10.6 samples/s
|
| 494 |
+
Step 12970/50462 | Loss: 2.6992 | LR: 4.29e-04 | 5644.4ms/step | 10.6 samples/s
|
| 495 |
+
Step 12980/50462 | Loss: 2.6826 | LR: 4.29e-04 | 5646.4ms/step | 10.6 samples/s
|
| 496 |
+
Step 12990/50462 | Loss: 2.6624 | LR: 4.29e-04 | 5643.9ms/step | 10.6 samples/s
|
| 497 |
+
Step 13000/50462 | Loss: 2.7072 | LR: 4.29e-04 | 5644.5ms/step | 10.6 samples/s
|
| 498 |
+
Validation at step 13000: Loss = 2.7452, Perplexity = 15.57
|
| 499 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13000
|
| 500 |
+
Step 13010/50462 | Loss: 2.7087 | LR: 4.29e-04 | 8430.9ms/step | 7.1 samples/s
|
| 501 |
+
Step 13020/50462 | Loss: 2.7035 | LR: 4.29e-04 | 5636.0ms/step | 10.6 samples/s
|
| 502 |
+
Step 13030/50462 | Loss: 2.6612 | LR: 4.28e-04 | 5637.9ms/step | 10.6 samples/s
|
| 503 |
+
Step 13040/50462 | Loss: 2.6868 | LR: 4.28e-04 | 5637.7ms/step | 10.6 samples/s
|
| 504 |
+
Step 13050/50462 | Loss: 2.7020 | LR: 4.28e-04 | 5639.0ms/step | 10.6 samples/s
|
| 505 |
+
Step 13060/50462 | Loss: 2.6835 | LR: 4.28e-04 | 5640.6ms/step | 10.6 samples/s
|
| 506 |
+
Step 13070/50462 | Loss: 2.6975 | LR: 4.28e-04 | 5640.5ms/step | 10.6 samples/s
|
| 507 |
+
Step 13080/50462 | Loss: 2.6663 | LR: 4.28e-04 | 5638.7ms/step | 10.6 samples/s
|
| 508 |
+
Step 13090/50462 | Loss: 2.6980 | LR: 4.28e-04 | 5638.9ms/step | 10.6 samples/s
|
| 509 |
+
Step 13100/50462 | Loss: 2.7095 | LR: 4.28e-04 | 5640.0ms/step | 10.6 samples/s
|
| 510 |
+
Step 13110/50462 | Loss: 2.6990 | LR: 4.28e-04 | 5640.7ms/step | 10.6 samples/s
|
| 511 |
+
Step 13120/50462 | Loss: 2.6762 | LR: 4.27e-04 | 5638.4ms/step | 10.6 samples/s
|
| 512 |
+
Step 13130/50462 | Loss: 2.6855 | LR: 4.27e-04 | 5637.8ms/step | 10.6 samples/s
|
| 513 |
+
Step 13140/50462 | Loss: 2.6784 | LR: 4.27e-04 | 5637.3ms/step | 10.6 samples/s
|
| 514 |
+
Step 13150/50462 | Loss: 2.6935 | LR: 4.27e-04 | 5639.2ms/step | 10.6 samples/s
|
| 515 |
+
Step 13160/50462 | Loss: 2.7033 | LR: 4.27e-04 | 5637.5ms/step | 10.6 samples/s
|
| 516 |
+
Step 13170/50462 | Loss: 2.6324 | LR: 4.27e-04 | 5639.1ms/step | 10.6 samples/s
|
| 517 |
+
Step 13180/50462 | Loss: 2.7152 | LR: 4.27e-04 | 5640.2ms/step | 10.6 samples/s
|
| 518 |
+
Step 13190/50462 | Loss: 2.6310 | LR: 4.27e-04 | 5639.8ms/step | 10.6 samples/s
|
| 519 |
+
Step 13200/50462 | Loss: 2.7208 | LR: 4.27e-04 | 5641.2ms/step | 10.6 samples/s
|
| 520 |
+
Step 13210/50462 | Loss: 2.6577 | LR: 4.26e-04 | 5642.0ms/step | 10.6 samples/s
|
| 521 |
+
Step 13220/50462 | Loss: 2.7141 | LR: 4.26e-04 | 5641.1ms/step | 10.6 samples/s
|
| 522 |
+
Step 13230/50462 | Loss: 2.6278 | LR: 4.26e-04 | 5639.8ms/step | 10.6 samples/s
|
| 523 |
+
Step 13240/50462 | Loss: 2.7089 | LR: 4.26e-04 | 5638.2ms/step | 10.6 samples/s
|
| 524 |
+
Step 13250/50462 | Loss: 2.6795 | LR: 4.26e-04 | 5634.8ms/step | 10.6 samples/s
|
| 525 |
+
Validation at step 13250: Loss = 2.7395, Perplexity = 15.48
|
| 526 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13250
|
| 527 |
+
Step 13260/50462 | Loss: 2.6344 | LR: 4.26e-04 | 8390.3ms/step | 7.2 samples/s
|
| 528 |
+
Step 13270/50462 | Loss: 2.6942 | LR: 4.26e-04 | 5635.6ms/step | 10.6 samples/s
|
| 529 |
+
Step 13280/50462 | Loss: 2.6995 | LR: 4.26e-04 | 5636.4ms/step | 10.6 samples/s
|
| 530 |
+
Step 13290/50462 | Loss: 2.6941 | LR: 4.26e-04 | 5639.4ms/step | 10.6 samples/s
|
| 531 |
+
Step 13300/50462 | Loss: 2.6853 | LR: 4.25e-04 | 5637.6ms/step | 10.6 samples/s
|
| 532 |
+
Step 13310/50462 | Loss: 2.6773 | LR: 4.25e-04 | 5637.8ms/step | 10.6 samples/s
|
| 533 |
+
Step 13320/50462 | Loss: 2.6918 | LR: 4.25e-04 | 5638.3ms/step | 10.6 samples/s
|
| 534 |
+
Step 13330/50462 | Loss: 2.7092 | LR: 4.25e-04 | 5638.8ms/step | 10.6 samples/s
|
| 535 |
+
Step 13340/50462 | Loss: 2.6727 | LR: 4.25e-04 | 5638.8ms/step | 10.6 samples/s
|
| 536 |
+
Step 13350/50462 | Loss: 2.6845 | LR: 4.25e-04 | 5637.5ms/step | 10.6 samples/s
|
| 537 |
+
Step 13360/50462 | Loss: 2.6827 | LR: 4.25e-04 | 5637.6ms/step | 10.6 samples/s
|
| 538 |
+
Step 13370/50462 | Loss: 2.7254 | LR: 4.25e-04 | 5638.6ms/step | 10.6 samples/s
|
| 539 |
+
Step 13380/50462 | Loss: 2.6825 | LR: 4.25e-04 | 5636.3ms/step | 10.6 samples/s
|
| 540 |
+
Step 13390/50462 | Loss: 2.6745 | LR: 4.24e-04 | 5637.4ms/step | 10.6 samples/s
|
| 541 |
+
Step 13400/50462 | Loss: 2.7167 | LR: 4.24e-04 | 5633.9ms/step | 10.6 samples/s
|
| 542 |
+
Step 13410/50462 | Loss: 2.6443 | LR: 4.24e-04 | 5637.3ms/step | 10.6 samples/s
|
| 543 |
+
Step 13420/50462 | Loss: 2.6787 | LR: 4.24e-04 | 5639.8ms/step | 10.6 samples/s
|
| 544 |
+
Step 13430/50462 | Loss: 2.7108 | LR: 4.24e-04 | 5635.9ms/step | 10.6 samples/s
|
| 545 |
+
Step 13440/50462 | Loss: 2.6802 | LR: 4.24e-04 | 5641.9ms/step | 10.6 samples/s
|
| 546 |
+
Step 13450/50462 | Loss: 2.7312 | LR: 4.24e-04 | 5638.6ms/step | 10.6 samples/s
|
| 547 |
+
Step 13460/50462 | Loss: 2.6788 | LR: 4.24e-04 | 5636.7ms/step | 10.6 samples/s
|
| 548 |
+
Step 13470/50462 | Loss: 2.6803 | LR: 4.23e-04 | 5639.1ms/step | 10.6 samples/s
|
| 549 |
+
Step 13480/50462 | Loss: 2.6908 | LR: 4.23e-04 | 5639.0ms/step | 10.6 samples/s
|
| 550 |
+
Step 13490/50462 | Loss: 2.6883 | LR: 4.23e-04 | 5639.8ms/step | 10.6 samples/s
|
| 551 |
+
Step 13500/50462 | Loss: 2.7029 | LR: 4.23e-04 | 5638.1ms/step | 10.6 samples/s
|
| 552 |
+
Validation at step 13500: Loss = 2.7361, Perplexity = 15.43
|
| 553 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-13500
|
| 554 |
+
Step 13510/50462 | Loss: 2.6593 | LR: 4.23e-04 | 8385.9ms/step | 7.2 samples/s
|
| 555 |
+
Step 13520/50462 | Loss: 2.7098 | LR: 4.23e-04 | 5628.9ms/step | 10.7 samples/s
|
| 556 |
+
Step 13530/50462 | Loss: 2.6718 | LR: 4.23e-04 | 5634.3ms/step | 10.6 samples/s
|
| 557 |
+
Step 13540/50462 | Loss: 2.6783 | LR: 4.23e-04 | 5638.9ms/step | 10.6 samples/s
|
| 558 |
+
Step 13550/50462 | Loss: 2.6606 | LR: 4.23e-04 | 5644.4ms/step | 10.6 samples/s
|
| 559 |
+
Step 13560/50462 | Loss: 2.6996 | LR: 4.22e-04 | 5644.4ms/step | 10.6 samples/s
|
| 560 |
+
Step 13570/50462 | Loss: 2.6735 | LR: 4.22e-04 | 5645.1ms/step | 10.6 samples/s
|
| 561 |
+
Step 13580/50462 | Loss: 2.6864 | LR: 4.22e-04 | 5645.7ms/step | 10.6 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed70_6gpu_resume41750.log
ADDED
|
@@ -0,0 +1,1186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0827 02:22:54.057000 172 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0827 02:22:54.057000 172 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0827 02:22:54.057000 172 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0827 02:22:54.057000 172 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 resume41750 6gpu",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed71"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: FalseCheckpoint activations: FalseCheckpoint activations: False
|
| 69 |
+
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750
|
| 74 |
+
Checkpoint activations: False
|
| 75 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
================================================================================
|
| 79 |
+
MODEL ARCHITECTURE
|
| 80 |
+
================================================================================
|
| 81 |
+
Model config:
|
| 82 |
+
vocab_size: 151936
|
| 83 |
+
hidden_size: 1536
|
| 84 |
+
num_hidden_layers: 28
|
| 85 |
+
num_attention_heads: 12
|
| 86 |
+
num_key_value_heads: 4
|
| 87 |
+
head_dim: 128
|
| 88 |
+
intermediate_size: 6144
|
| 89 |
+
max_position_embeddings: 32768
|
| 90 |
+
rope_theta: 1000000
|
| 91 |
+
rope_train_position_scale_set: None
|
| 92 |
+
attention_type: sparse_stochastic
|
| 93 |
+
sparse_block_size: 128
|
| 94 |
+
sparse_num_random_blocks: 31
|
| 95 |
+
sparse_mask_pool_size: 1
|
| 96 |
+
sparse_max_seq_length: 8192
|
| 97 |
+
|
| 98 |
+
Parameter count:
|
| 99 |
+
Total parameters: 1,202,352,640
|
| 100 |
+
Trainable parameters: 1,202,352,640
|
| 101 |
+
Non-trainable parameters: 0
|
| 102 |
+
Model size (BF16): 2.40 GB
|
| 103 |
+
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
LAYER-BY-LAYER SUMMARY
|
| 106 |
+
--------------------------------------------------------------------------------
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 109 |
+
================================================================================================================================================================
|
| 110 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 111 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 112 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 113 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 143 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 144 |
+
================================================================================================================================================================
|
| 145 |
+
Total params: 1,435,726,336
|
| 146 |
+
Trainable params: 1,435,726,336
|
| 147 |
+
Non-trainable params: 0
|
| 148 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 149 |
+
================================================================================================================================================================
|
| 150 |
+
Input size (MB): 0.07
|
| 151 |
+
Forward/backward pass size (MB): 26218.59
|
| 152 |
+
Params size (MB): 2871.45
|
| 153 |
+
Estimated Total Size (MB): 29090.11
|
| 154 |
+
================================================================================================================================================================
|
| 155 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 156 |
+
================================================================================
|
| 157 |
+
|
| 158 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
|
| 163 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 164 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 165 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 166 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 167 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 168 |
+
|
| 169 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 170 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 171 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 172 |
+
Dataset split: 3027764 train, 159356 validation
|
| 173 |
+
Training examples (packed, train split): 3027764
|
| 174 |
+
Validation examples (packed, val split): 159356
|
| 175 |
+
Per-device batch size: 1
|
| 176 |
+
World size (GPUs): 6
|
| 177 |
+
Gradient accumulation steps: 10
|
| 178 |
+
Effective batch size (seqs/step): 60
|
| 179 |
+
Sequence length: 8192
|
| 180 |
+
Tokens per optimizer step: 491,520
|
| 181 |
+
Micro-batches per epoch (per rank): 504628
|
| 182 |
+
Optimizer steps per epoch: 50462
|
| 183 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 184 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750...
|
| 185 |
+
Preserving checkpoint LR state for exact resume: current=[3.694238556405591e-05], schedule_base=[0.0005]
|
| 186 |
+
Resumed from checkpoint at step 41750 (epoch 0, micro_step 122500)
|
| 187 |
+
Resuming training from step 41750
|
| 188 |
+
Resuming wandb run rgmkfpnl
|
| 189 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 190 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 191 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 192 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_022420-rgmkfpnl
|
| 193 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 194 |
+
wandb: Resuming run 1.2b dense 8k seed70 resume41750 6gpu
|
| 195 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 196 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 197 |
+
Starting training...
|
| 198 |
+
Validation at step 41750: Loss = 2.4998, Perplexity = 12.18
|
| 199 |
+
wandb: WARNING Tried to log to step 41750 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Switching attention mode for step 41751: dense (dense step interval)
|
| 201 |
+
Step 41760/50462 | Loss: 2.4765 | LR: 3.69e-05 | 27125.5ms/step | 2.2 samples/s
|
| 202 |
+
wandb: WARNING Tried to log to step 41760 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 203 |
+
Step 41770/50462 | Loss: 2.4496 | LR: 3.68e-05 | 1910.1ms/step | 31.4 samples/s
|
| 204 |
+
wandb: WARNING Tried to log to step 41770 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 205 |
+
Step 41780/50462 | Loss: 2.4236 | LR: 3.67e-05 | 1914.1ms/step | 31.3 samples/s
|
| 206 |
+
wandb: WARNING Tried to log to step 41780 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 207 |
+
Step 41790/50462 | Loss: 2.4385 | LR: 3.66e-05 | 1913.0ms/step | 31.4 samples/s
|
| 208 |
+
wandb: WARNING Tried to log to step 41790 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 209 |
+
Step 41800/50462 | Loss: 2.4967 | LR: 3.65e-05 | 1914.8ms/step | 31.3 samples/s
|
| 210 |
+
wandb: WARNING Tried to log to step 41800 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 211 |
+
Step 41810/50462 | Loss: 2.4740 | LR: 3.64e-05 | 1916.3ms/step | 31.3 samples/s
|
| 212 |
+
wandb: WARNING Tried to log to step 41810 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 213 |
+
Step 41820/50462 | Loss: 2.4658 | LR: 3.64e-05 | 1915.8ms/step | 31.3 samples/s
|
| 214 |
+
wandb: WARNING Tried to log to step 41820 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Step 41830/50462 | Loss: 2.4844 | LR: 3.63e-05 | 1917.8ms/step | 31.3 samples/s
|
| 216 |
+
wandb: WARNING Tried to log to step 41830 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 217 |
+
Step 41840/50462 | Loss: 2.4547 | LR: 3.62e-05 | 1918.1ms/step | 31.3 samples/s
|
| 218 |
+
wandb: WARNING Tried to log to step 41840 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 219 |
+
Step 41850/50462 | Loss: 2.4263 | LR: 3.61e-05 | 1918.1ms/step | 31.3 samples/s
|
| 220 |
+
wandb: WARNING Tried to log to step 41850 that is less than the current step 41851. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 221 |
+
Step 41860/50462 | Loss: 2.4356 | LR: 3.60e-05 | 1917.3ms/step | 31.3 samples/s
|
| 222 |
+
Step 41870/50462 | Loss: 2.4848 | LR: 3.60e-05 | 1919.7ms/step | 31.3 samples/s
|
| 223 |
+
Step 41880/50462 | Loss: 2.4195 | LR: 3.59e-05 | 1920.1ms/step | 31.2 samples/s
|
| 224 |
+
Step 41890/50462 | Loss: 2.4288 | LR: 3.58e-05 | 1918.8ms/step | 31.3 samples/s
|
| 225 |
+
Step 41900/50462 | Loss: 2.4447 | LR: 3.57e-05 | 1920.3ms/step | 31.2 samples/s
|
| 226 |
+
Step 41910/50462 | Loss: 2.4391 | LR: 3.56e-05 | 1919.8ms/step | 31.3 samples/s
|
| 227 |
+
Step 41920/50462 | Loss: 2.5013 | LR: 3.55e-05 | 1919.4ms/step | 31.3 samples/s
|
| 228 |
+
Step 41930/50462 | Loss: 2.4701 | LR: 3.55e-05 | 1918.1ms/step | 31.3 samples/s
|
| 229 |
+
Step 41940/50462 | Loss: 2.4601 | LR: 3.54e-05 | 1919.5ms/step | 31.3 samples/s
|
| 230 |
+
Step 41950/50462 | Loss: 2.4752 | LR: 3.53e-05 | 1918.9ms/step | 31.3 samples/s
|
| 231 |
+
Step 41960/50462 | Loss: 2.4323 | LR: 3.52e-05 | 1919.8ms/step | 31.3 samples/s
|
| 232 |
+
Step 41970/50462 | Loss: 2.4556 | LR: 3.51e-05 | 1918.4ms/step | 31.3 samples/s
|
| 233 |
+
Step 41980/50462 | Loss: 2.4609 | LR: 3.51e-05 | 1919.0ms/step | 31.3 samples/s
|
| 234 |
+
Step 41990/50462 | Loss: 2.4705 | LR: 3.50e-05 | 1917.8ms/step | 31.3 samples/s
|
| 235 |
+
Step 42000/50462 | Loss: 2.4515 | LR: 3.49e-05 | 1918.7ms/step | 31.3 samples/s
|
| 236 |
+
Validation at step 42000: Loss = 2.4997, Perplexity = 12.18
|
| 237 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42000
|
| 238 |
+
Step 42010/50462 | Loss: 2.4335 | LR: 3.48e-05 | 3284.3ms/step | 18.3 samples/s
|
| 239 |
+
Step 42020/50462 | Loss: 2.4252 | LR: 3.47e-05 | 1918.8ms/step | 31.3 samples/s
|
| 240 |
+
Step 42030/50462 | Loss: 2.4807 | LR: 3.47e-05 | 1919.7ms/step | 31.3 samples/s
|
| 241 |
+
Step 42040/50462 | Loss: 2.4371 | LR: 3.46e-05 | 1921.0ms/step | 31.2 samples/s
|
| 242 |
+
Step 42050/50462 | Loss: 2.4613 | LR: 3.45e-05 | 1920.8ms/step | 31.2 samples/s
|
| 243 |
+
Step 42060/50462 | Loss: 2.4550 | LR: 3.44e-05 | 1923.5ms/step | 31.2 samples/s
|
| 244 |
+
Step 42070/50462 | Loss: 2.4377 | LR: 3.43e-05 | 1921.2ms/step | 31.2 samples/s
|
| 245 |
+
Step 42080/50462 | Loss: 2.4887 | LR: 3.43e-05 | 1921.2ms/step | 31.2 samples/s
|
| 246 |
+
Step 42090/50462 | Loss: 2.4232 | LR: 3.42e-05 | 1922.5ms/step | 31.2 samples/s
|
| 247 |
+
Step 42100/50462 | Loss: 2.4605 | LR: 3.41e-05 | 1922.8ms/step | 31.2 samples/s
|
| 248 |
+
Step 42110/50462 | Loss: 2.4762 | LR: 3.40e-05 | 1919.4ms/step | 31.3 samples/s
|
| 249 |
+
Step 42120/50462 | Loss: 2.4436 | LR: 3.39e-05 | 1924.1ms/step | 31.2 samples/s
|
| 250 |
+
Step 42130/50462 | Loss: 2.4249 | LR: 3.39e-05 | 1923.5ms/step | 31.2 samples/s
|
| 251 |
+
Step 42140/50462 | Loss: 2.4420 | LR: 3.38e-05 | 1922.4ms/step | 31.2 samples/s
|
| 252 |
+
Step 42150/50462 | Loss: 2.4723 | LR: 3.37e-05 | 1921.3ms/step | 31.2 samples/s
|
| 253 |
+
Step 42160/50462 | Loss: 2.4360 | LR: 3.36e-05 | 1922.5ms/step | 31.2 samples/s
|
| 254 |
+
Step 42170/50462 | Loss: 2.4758 | LR: 3.35e-05 | 1923.2ms/step | 31.2 samples/s
|
| 255 |
+
Step 42180/50462 | Loss: 2.4251 | LR: 3.35e-05 | 1922.6ms/step | 31.2 samples/s
|
| 256 |
+
Step 42190/50462 | Loss: 2.4669 | LR: 3.34e-05 | 1922.1ms/step | 31.2 samples/s
|
| 257 |
+
Step 42200/50462 | Loss: 2.4345 | LR: 3.33e-05 | 1923.9ms/step | 31.2 samples/s
|
| 258 |
+
Step 42210/50462 | Loss: 2.4191 | LR: 3.32e-05 | 1923.9ms/step | 31.2 samples/s
|
| 259 |
+
Step 42220/50462 | Loss: 2.4376 | LR: 3.32e-05 | 1925.2ms/step | 31.2 samples/s
|
| 260 |
+
Step 42230/50462 | Loss: 2.4225 | LR: 3.31e-05 | 1924.5ms/step | 31.2 samples/s
|
| 261 |
+
Step 42240/50462 | Loss: 2.4419 | LR: 3.30e-05 | 1923.8ms/step | 31.2 samples/s
|
| 262 |
+
Step 42250/50462 | Loss: 2.4459 | LR: 3.29e-05 | 1923.2ms/step | 31.2 samples/s
|
| 263 |
+
Validation at step 42250: Loss = 2.4996, Perplexity = 12.18
|
| 264 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42250
|
| 265 |
+
Step 42260/50462 | Loss: 2.4690 | LR: 3.28e-05 | 3288.4ms/step | 18.2 samples/s
|
| 266 |
+
Step 42270/50462 | Loss: 2.4109 | LR: 3.28e-05 | 1918.5ms/step | 31.3 samples/s
|
| 267 |
+
Step 42280/50462 | Loss: 2.4810 | LR: 3.27e-05 | 1918.2ms/step | 31.3 samples/s
|
| 268 |
+
Step 42290/50462 | Loss: 2.4652 | LR: 3.26e-05 | 1920.9ms/step | 31.2 samples/s
|
| 269 |
+
Step 42300/50462 | Loss: 2.4609 | LR: 3.25e-05 | 1920.8ms/step | 31.2 samples/s
|
| 270 |
+
Step 42310/50462 | Loss: 2.4420 | LR: 3.24e-05 | 1921.5ms/step | 31.2 samples/s
|
| 271 |
+
Step 42320/50462 | Loss: 2.4595 | LR: 3.24e-05 | 1920.7ms/step | 31.2 samples/s
|
| 272 |
+
Step 42330/50462 | Loss: 2.4655 | LR: 3.23e-05 | 1921.4ms/step | 31.2 samples/s
|
| 273 |
+
Step 42340/50462 | Loss: 2.4250 | LR: 3.22e-05 | 1921.4ms/step | 31.2 samples/s
|
| 274 |
+
Step 42350/50462 | Loss: 2.4435 | LR: 3.21e-05 | 1921.1ms/step | 31.2 samples/s
|
| 275 |
+
Step 42360/50462 | Loss: 2.4586 | LR: 3.21e-05 | 1921.8ms/step | 31.2 samples/s
|
| 276 |
+
Step 42370/50462 | Loss: 2.4205 | LR: 3.20e-05 | 1923.9ms/step | 31.2 samples/s
|
| 277 |
+
Step 42380/50462 | Loss: 2.4001 | LR: 3.19e-05 | 1920.1ms/step | 31.2 samples/s
|
| 278 |
+
Step 42390/50462 | Loss: 2.4502 | LR: 3.18e-05 | 1919.1ms/step | 31.3 samples/s
|
| 279 |
+
Step 42400/50462 | Loss: 2.4204 | LR: 3.18e-05 | 1922.4ms/step | 31.2 samples/s
|
| 280 |
+
Step 42410/50462 | Loss: 2.4973 | LR: 3.17e-05 | 1922.7ms/step | 31.2 samples/s
|
| 281 |
+
Step 42420/50462 | Loss: 2.4565 | LR: 3.16e-05 | 1922.9ms/step | 31.2 samples/s
|
| 282 |
+
Step 42430/50462 | Loss: 2.4520 | LR: 3.15e-05 | 1925.3ms/step | 31.2 samples/s
|
| 283 |
+
Step 42440/50462 | Loss: 2.4613 | LR: 3.14e-05 | 1924.3ms/step | 31.2 samples/s
|
| 284 |
+
Step 42450/50462 | Loss: 2.4724 | LR: 3.14e-05 | 1922.9ms/step | 31.2 samples/s
|
| 285 |
+
Step 42460/50462 | Loss: 2.4924 | LR: 3.13e-05 | 1923.1ms/step | 31.2 samples/s
|
| 286 |
+
Step 42470/50462 | Loss: 2.4884 | LR: 3.12e-05 | 1922.4ms/step | 31.2 samples/s
|
| 287 |
+
Step 42480/50462 | Loss: 2.4620 | LR: 3.11e-05 | 1920.3ms/step | 31.2 samples/s
|
| 288 |
+
Step 42490/50462 | Loss: 2.4762 | LR: 3.11e-05 | 1924.7ms/step | 31.2 samples/s
|
| 289 |
+
Step 42500/50462 | Loss: 2.4652 | LR: 3.10e-05 | 1924.8ms/step | 31.2 samples/s
|
| 290 |
+
Validation at step 42500: Loss = 2.4997, Perplexity = 12.18
|
| 291 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42500
|
| 292 |
+
Step 42510/50462 | Loss: 2.4799 | LR: 3.09e-05 | 3280.6ms/step | 18.3 samples/s
|
| 293 |
+
Step 42520/50462 | Loss: 2.4559 | LR: 3.08e-05 | 1918.6ms/step | 31.3 samples/s
|
| 294 |
+
Step 42530/50462 | Loss: 2.4718 | LR: 3.08e-05 | 1916.3ms/step | 31.3 samples/s
|
| 295 |
+
Step 42540/50462 | Loss: 2.4174 | LR: 3.07e-05 | 1913.1ms/step | 31.4 samples/s
|
| 296 |
+
Step 42550/50462 | Loss: 2.4554 | LR: 3.06e-05 | 1915.9ms/step | 31.3 samples/s
|
| 297 |
+
Step 42560/50462 | Loss: 2.4561 | LR: 3.05e-05 | 1917.2ms/step | 31.3 samples/s
|
| 298 |
+
Step 42570/50462 | Loss: 2.4430 | LR: 3.05e-05 | 1916.8ms/step | 31.3 samples/s
|
| 299 |
+
Step 42580/50462 | Loss: 2.4743 | LR: 3.04e-05 | 1916.4ms/step | 31.3 samples/s
|
| 300 |
+
Step 42590/50462 | Loss: 2.4770 | LR: 3.03e-05 | 1916.3ms/step | 31.3 samples/s
|
| 301 |
+
Step 42600/50462 | Loss: 2.4816 | LR: 3.02e-05 | 1918.2ms/step | 31.3 samples/s
|
| 302 |
+
Step 42610/50462 | Loss: 2.4614 | LR: 3.02e-05 | 1916.3ms/step | 31.3 samples/s
|
| 303 |
+
Step 42620/50462 | Loss: 2.4648 | LR: 3.01e-05 | 1916.7ms/step | 31.3 samples/s
|
| 304 |
+
Step 42630/50462 | Loss: 2.4493 | LR: 3.00e-05 | 1917.4ms/step | 31.3 samples/s
|
| 305 |
+
Step 42640/50462 | Loss: 2.4549 | LR: 2.99e-05 | 1919.6ms/step | 31.3 samples/s
|
| 306 |
+
Step 42650/50462 | Loss: 2.4691 | LR: 2.99e-05 | 1917.3ms/step | 31.3 samples/s
|
| 307 |
+
Step 42660/50462 | Loss: 2.4372 | LR: 2.98e-05 | 1916.8ms/step | 31.3 samples/s
|
| 308 |
+
Step 42670/50462 | Loss: 2.4466 | LR: 2.97e-05 | 1914.5ms/step | 31.3 samples/s
|
| 309 |
+
Step 42680/50462 | Loss: 2.4520 | LR: 2.96e-05 | 1916.4ms/step | 31.3 samples/s
|
| 310 |
+
Step 42690/50462 | Loss: 2.4741 | LR: 2.96e-05 | 1917.8ms/step | 31.3 samples/s
|
| 311 |
+
Step 42700/50462 | Loss: 2.4279 | LR: 2.95e-05 | 1916.4ms/step | 31.3 samples/s
|
| 312 |
+
Step 42710/50462 | Loss: 2.4447 | LR: 2.94e-05 | 1919.1ms/step | 31.3 samples/s
|
| 313 |
+
Step 42720/50462 | Loss: 2.4124 | LR: 2.93e-05 | 1918.8ms/step | 31.3 samples/s
|
| 314 |
+
Step 42730/50462 | Loss: 2.4278 | LR: 2.93e-05 | 1918.3ms/step | 31.3 samples/s
|
| 315 |
+
Step 42740/50462 | Loss: 2.4914 | LR: 2.92e-05 | 1917.6ms/step | 31.3 samples/s
|
| 316 |
+
Step 42750/50462 | Loss: 2.4426 | LR: 2.91e-05 | 1918.2ms/step | 31.3 samples/s
|
| 317 |
+
Validation at step 42750: Loss = 2.4997, Perplexity = 12.18
|
| 318 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-42750
|
| 319 |
+
Step 42760/50462 | Loss: 2.4383 | LR: 2.90e-05 | 3276.0ms/step | 18.3 samples/s
|
| 320 |
+
Step 42770/50462 | Loss: 2.4460 | LR: 2.90e-05 | 1913.9ms/step | 31.3 samples/s
|
| 321 |
+
Step 42780/50462 | Loss: 2.4482 | LR: 2.89e-05 | 1918.4ms/step | 31.3 samples/s
|
| 322 |
+
Step 42790/50462 | Loss: 2.4689 | LR: 2.88e-05 | 1922.2ms/step | 31.2 samples/s
|
| 323 |
+
Step 42800/50462 | Loss: 2.4269 | LR: 2.87e-05 | 1914.0ms/step | 31.3 samples/s
|
| 324 |
+
Step 42810/50462 | Loss: 2.4772 | LR: 2.87e-05 | 1917.5ms/step | 31.3 samples/s
|
| 325 |
+
Step 42820/50462 | Loss: 2.4598 | LR: 2.86e-05 | 1916.4ms/step | 31.3 samples/s
|
| 326 |
+
Step 42830/50462 | Loss: 2.4822 | LR: 2.85e-05 | 1917.5ms/step | 31.3 samples/s
|
| 327 |
+
Step 42840/50462 | Loss: 2.4376 | LR: 2.84e-05 | 1917.5ms/step | 31.3 samples/s
|
| 328 |
+
Step 42850/50462 | Loss: 2.4690 | LR: 2.84e-05 | 1915.6ms/step | 31.3 samples/s
|
| 329 |
+
Step 42860/50462 | Loss: 2.4457 | LR: 2.83e-05 | 1915.3ms/step | 31.3 samples/s
|
| 330 |
+
Step 42870/50462 | Loss: 2.4663 | LR: 2.82e-05 | 1918.1ms/step | 31.3 samples/s
|
| 331 |
+
Step 42880/50462 | Loss: 2.4254 | LR: 2.82e-05 | 1921.7ms/step | 31.2 samples/s
|
| 332 |
+
Step 42890/50462 | Loss: 2.4646 | LR: 2.81e-05 | 1917.2ms/step | 31.3 samples/s
|
| 333 |
+
Step 42900/50462 | Loss: 2.4320 | LR: 2.80e-05 | 1920.1ms/step | 31.2 samples/s
|
| 334 |
+
Step 42910/50462 | Loss: 2.4421 | LR: 2.79e-05 | 1920.4ms/step | 31.2 samples/s
|
| 335 |
+
Step 42920/50462 | Loss: 2.4584 | LR: 2.79e-05 | 1921.0ms/step | 31.2 samples/s
|
| 336 |
+
Step 42930/50462 | Loss: 2.4675 | LR: 2.78e-05 | 1917.1ms/step | 31.3 samples/s
|
| 337 |
+
Step 42940/50462 | Loss: 2.4397 | LR: 2.77e-05 | 1918.4ms/step | 31.3 samples/s
|
| 338 |
+
Step 42950/50462 | Loss: 2.4409 | LR: 2.76e-05 | 1917.7ms/step | 31.3 samples/s
|
| 339 |
+
Step 42960/50462 | Loss: 2.4326 | LR: 2.76e-05 | 1920.4ms/step | 31.2 samples/s
|
| 340 |
+
Step 42970/50462 | Loss: 2.4749 | LR: 2.75e-05 | 1919.4ms/step | 31.3 samples/s
|
| 341 |
+
Step 42980/50462 | Loss: 2.4205 | LR: 2.74e-05 | 1915.8ms/step | 31.3 samples/s
|
| 342 |
+
Step 42990/50462 | Loss: 2.4393 | LR: 2.74e-05 | 1917.7ms/step | 31.3 samples/s
|
| 343 |
+
Step 43000/50462 | Loss: 2.4520 | LR: 2.73e-05 | 1917.1ms/step | 31.3 samples/s
|
| 344 |
+
Validation at step 43000: Loss = 2.4995, Perplexity = 12.18
|
| 345 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43000
|
| 346 |
+
Step 43010/50462 | Loss: 2.4904 | LR: 2.72e-05 | 3275.3ms/step | 18.3 samples/s
|
| 347 |
+
Step 43020/50462 | Loss: 2.4611 | LR: 2.71e-05 | 1917.3ms/step | 31.3 samples/s
|
| 348 |
+
Step 43030/50462 | Loss: 2.4958 | LR: 2.71e-05 | 1917.3ms/step | 31.3 samples/s
|
| 349 |
+
Step 43040/50462 | Loss: 2.4301 | LR: 2.70e-05 | 1919.0ms/step | 31.3 samples/s
|
| 350 |
+
Step 43050/50462 | Loss: 2.4283 | LR: 2.69e-05 | 1918.2ms/step | 31.3 samples/s
|
| 351 |
+
Step 43060/50462 | Loss: 2.4670 | LR: 2.69e-05 | 1919.3ms/step | 31.3 samples/s
|
| 352 |
+
Step 43070/50462 | Loss: 2.4626 | LR: 2.68e-05 | 1918.4ms/step | 31.3 samples/s
|
| 353 |
+
Step 43080/50462 | Loss: 2.4339 | LR: 2.67e-05 | 1916.9ms/step | 31.3 samples/s
|
| 354 |
+
Step 43090/50462 | Loss: 2.4369 | LR: 2.66e-05 | 1917.3ms/step | 31.3 samples/s
|
| 355 |
+
Step 43100/50462 | Loss: 2.4060 | LR: 2.66e-05 | 1916.0ms/step | 31.3 samples/s
|
| 356 |
+
Step 43110/50462 | Loss: 2.4322 | LR: 2.65e-05 | 1916.7ms/step | 31.3 samples/s
|
| 357 |
+
Step 43120/50462 | Loss: 2.4421 | LR: 2.64e-05 | 1920.8ms/step | 31.2 samples/s
|
| 358 |
+
Step 43130/50462 | Loss: 2.4280 | LR: 2.64e-05 | 1918.3ms/step | 31.3 samples/s
|
| 359 |
+
Step 43140/50462 | Loss: 2.4364 | LR: 2.63e-05 | 1915.2ms/step | 31.3 samples/s
|
| 360 |
+
Step 43150/50462 | Loss: 2.4483 | LR: 2.62e-05 | 1921.7ms/step | 31.2 samples/s
|
| 361 |
+
Step 43160/50462 | Loss: 2.4592 | LR: 2.61e-05 | 1924.6ms/step | 31.2 samples/s
|
| 362 |
+
Step 43170/50462 | Loss: 2.4678 | LR: 2.61e-05 | 1921.7ms/step | 31.2 samples/s
|
| 363 |
+
Step 43180/50462 | Loss: 2.4234 | LR: 2.60e-05 | 1917.6ms/step | 31.3 samples/s
|
| 364 |
+
Step 43190/50462 | Loss: 2.4756 | LR: 2.59e-05 | 1919.0ms/step | 31.3 samples/s
|
| 365 |
+
Step 43200/50462 | Loss: 2.4294 | LR: 2.59e-05 | 1918.6ms/step | 31.3 samples/s
|
| 366 |
+
Step 43210/50462 | Loss: 2.4174 | LR: 2.58e-05 | 1927.6ms/step | 31.1 samples/s
|
| 367 |
+
Step 43220/50462 | Loss: 2.4747 | LR: 2.57e-05 | 1917.4ms/step | 31.3 samples/s
|
| 368 |
+
Step 43230/50462 | Loss: 2.4487 | LR: 2.57e-05 | 1920.2ms/step | 31.2 samples/s
|
| 369 |
+
Step 43240/50462 | Loss: 2.4113 | LR: 2.56e-05 | 1918.2ms/step | 31.3 samples/s
|
| 370 |
+
Step 43250/50462 | Loss: 2.4730 | LR: 2.55e-05 | 1916.9ms/step | 31.3 samples/s
|
| 371 |
+
Validation at step 43250: Loss = 2.4995, Perplexity = 12.18
|
| 372 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43250
|
| 373 |
+
Step 43260/50462 | Loss: 2.4482 | LR: 2.54e-05 | 3346.3ms/step | 17.9 samples/s
|
| 374 |
+
Step 43270/50462 | Loss: 2.4502 | LR: 2.54e-05 | 1917.0ms/step | 31.3 samples/s
|
| 375 |
+
Step 43280/50462 | Loss: 2.4405 | LR: 2.53e-05 | 1917.3ms/step | 31.3 samples/s
|
| 376 |
+
Step 43290/50462 | Loss: 2.4461 | LR: 2.52e-05 | 1916.4ms/step | 31.3 samples/s
|
| 377 |
+
Step 43300/50462 | Loss: 2.4342 | LR: 2.52e-05 | 1916.8ms/step | 31.3 samples/s
|
| 378 |
+
Step 43310/50462 | Loss: 2.4266 | LR: 2.51e-05 | 1917.0ms/step | 31.3 samples/s
|
| 379 |
+
Step 43320/50462 | Loss: 2.4717 | LR: 2.50e-05 | 1923.6ms/step | 31.2 samples/s
|
| 380 |
+
Step 43330/50462 | Loss: 2.4409 | LR: 2.50e-05 | 1924.6ms/step | 31.2 samples/s
|
| 381 |
+
Step 43340/50462 | Loss: 2.4457 | LR: 2.49e-05 | 1922.3ms/step | 31.2 samples/s
|
| 382 |
+
Step 43350/50462 | Loss: 2.4038 | LR: 2.48e-05 | 1923.2ms/step | 31.2 samples/s
|
| 383 |
+
Step 43360/50462 | Loss: 2.4263 | LR: 2.48e-05 | 1922.0ms/step | 31.2 samples/s
|
| 384 |
+
Step 43370/50462 | Loss: 2.4600 | LR: 2.47e-05 | 1926.4ms/step | 31.1 samples/s
|
| 385 |
+
Step 43380/50462 | Loss: 2.4333 | LR: 2.46e-05 | 1922.0ms/step | 31.2 samples/s
|
| 386 |
+
Step 43390/50462 | Loss: 2.4387 | LR: 2.46e-05 | 1922.3ms/step | 31.2 samples/s
|
| 387 |
+
Step 43400/50462 | Loss: 2.4163 | LR: 2.45e-05 | 1927.5ms/step | 31.1 samples/s
|
| 388 |
+
Step 43410/50462 | Loss: 2.4875 | LR: 2.44e-05 | 1923.8ms/step | 31.2 samples/s
|
| 389 |
+
Step 43420/50462 | Loss: 2.4688 | LR: 2.44e-05 | 1922.6ms/step | 31.2 samples/s
|
| 390 |
+
Step 43430/50462 | Loss: 2.4565 | LR: 2.43e-05 | 1922.6ms/step | 31.2 samples/s
|
| 391 |
+
Step 43440/50462 | Loss: 2.4940 | LR: 2.42e-05 | 1925.5ms/step | 31.2 samples/s
|
| 392 |
+
Step 43450/50462 | Loss: 2.4718 | LR: 2.41e-05 | 1925.2ms/step | 31.2 samples/s
|
| 393 |
+
Step 43460/50462 | Loss: 2.4385 | LR: 2.41e-05 | 1924.7ms/step | 31.2 samples/s
|
| 394 |
+
Step 43470/50462 | Loss: 2.4937 | LR: 2.40e-05 | 1924.2ms/step | 31.2 samples/s
|
| 395 |
+
Step 43480/50462 | Loss: 2.4716 | LR: 2.39e-05 | 1922.1ms/step | 31.2 samples/s
|
| 396 |
+
Step 43490/50462 | Loss: 2.4598 | LR: 2.39e-05 | 1926.3ms/step | 31.1 samples/s
|
| 397 |
+
Step 43500/50462 | Loss: 2.4390 | LR: 2.38e-05 | 1926.8ms/step | 31.1 samples/s
|
| 398 |
+
Validation at step 43500: Loss = 2.4995, Perplexity = 12.18
|
| 399 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43500
|
| 400 |
+
Step 43510/50462 | Loss: 2.4165 | LR: 2.37e-05 | 3278.5ms/step | 18.3 samples/s
|
| 401 |
+
Step 43520/50462 | Loss: 2.4469 | LR: 2.37e-05 | 1916.2ms/step | 31.3 samples/s
|
| 402 |
+
Step 43530/50462 | Loss: 2.4435 | LR: 2.36e-05 | 1920.8ms/step | 31.2 samples/s
|
| 403 |
+
Step 43540/50462 | Loss: 2.4546 | LR: 2.35e-05 | 1914.7ms/step | 31.3 samples/s
|
| 404 |
+
Step 43550/50462 | Loss: 2.4757 | LR: 2.35e-05 | 1919.3ms/step | 31.3 samples/s
|
| 405 |
+
Step 43560/50462 | Loss: 2.4365 | LR: 2.34e-05 | 1919.7ms/step | 31.3 samples/s
|
| 406 |
+
Step 43570/50462 | Loss: 2.4800 | LR: 2.33e-05 | 1923.8ms/step | 31.2 samples/s
|
| 407 |
+
Step 43580/50462 | Loss: 2.4194 | LR: 2.33e-05 | 1923.8ms/step | 31.2 samples/s
|
| 408 |
+
Step 43590/50462 | Loss: 2.4146 | LR: 2.32e-05 | 1924.6ms/step | 31.2 samples/s
|
| 409 |
+
Step 43600/50462 | Loss: 2.4287 | LR: 2.31e-05 | 1923.0ms/step | 31.2 samples/s
|
| 410 |
+
Step 43610/50462 | Loss: 2.4385 | LR: 2.31e-05 | 1919.6ms/step | 31.3 samples/s
|
| 411 |
+
Step 43620/50462 | Loss: 2.4355 | LR: 2.30e-05 | 1917.5ms/step | 31.3 samples/s
|
| 412 |
+
Step 43630/50462 | Loss: 2.4303 | LR: 2.29e-05 | 1917.8ms/step | 31.3 samples/s
|
| 413 |
+
Step 43640/50462 | Loss: 2.4537 | LR: 2.29e-05 | 1918.3ms/step | 31.3 samples/s
|
| 414 |
+
Step 43650/50462 | Loss: 2.4379 | LR: 2.28e-05 | 1923.9ms/step | 31.2 samples/s
|
| 415 |
+
Step 43660/50462 | Loss: 2.4450 | LR: 2.27e-05 | 1922.6ms/step | 31.2 samples/s
|
| 416 |
+
Step 43670/50462 | Loss: 2.4526 | LR: 2.27e-05 | 1920.9ms/step | 31.2 samples/s
|
| 417 |
+
Step 43680/50462 | Loss: 2.3862 | LR: 2.26e-05 | 1923.7ms/step | 31.2 samples/s
|
| 418 |
+
Step 43690/50462 | Loss: 2.4481 | LR: 2.25e-05 | 1920.9ms/step | 31.2 samples/s
|
| 419 |
+
Step 43700/50462 | Loss: 2.4789 | LR: 2.25e-05 | 1923.0ms/step | 31.2 samples/s
|
| 420 |
+
Step 43710/50462 | Loss: 2.4246 | LR: 2.24e-05 | 1921.9ms/step | 31.2 samples/s
|
| 421 |
+
Step 43720/50462 | Loss: 2.4366 | LR: 2.24e-05 | 1921.3ms/step | 31.2 samples/s
|
| 422 |
+
Step 43730/50462 | Loss: 2.4544 | LR: 2.23e-05 | 1919.6ms/step | 31.3 samples/s
|
| 423 |
+
Step 43740/50462 | Loss: 2.4309 | LR: 2.22e-05 | 1917.4ms/step | 31.3 samples/s
|
| 424 |
+
Step 43750/50462 | Loss: 2.4466 | LR: 2.22e-05 | 1919.3ms/step | 31.3 samples/s
|
| 425 |
+
Validation at step 43750: Loss = 2.4995, Perplexity = 12.18
|
| 426 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-43750
|
| 427 |
+
Step 43760/50462 | Loss: 2.4365 | LR: 2.21e-05 | 3258.8ms/step | 18.4 samples/s
|
| 428 |
+
Step 43770/50462 | Loss: 2.4684 | LR: 2.20e-05 | 1921.9ms/step | 31.2 samples/s
|
| 429 |
+
Step 43780/50462 | Loss: 2.4653 | LR: 2.20e-05 | 1922.1ms/step | 31.2 samples/s
|
| 430 |
+
Step 43790/50462 | Loss: 2.4065 | LR: 2.19e-05 | 1918.8ms/step | 31.3 samples/s
|
| 431 |
+
Step 43800/50462 | Loss: 2.4566 | LR: 2.18e-05 | 1918.2ms/step | 31.3 samples/s
|
| 432 |
+
Step 43810/50462 | Loss: 2.4250 | LR: 2.18e-05 | 1917.9ms/step | 31.3 samples/s
|
| 433 |
+
Step 43820/50462 | Loss: 2.5246 | LR: 2.17e-05 | 1917.3ms/step | 31.3 samples/s
|
| 434 |
+
Step 43830/50462 | Loss: 2.4840 | LR: 2.16e-05 | 1916.8ms/step | 31.3 samples/s
|
| 435 |
+
Step 43840/50462 | Loss: 2.4121 | LR: 2.16e-05 | 1916.7ms/step | 31.3 samples/s
|
| 436 |
+
Step 43850/50462 | Loss: 2.3902 | LR: 2.15e-05 | 1917.1ms/step | 31.3 samples/s
|
| 437 |
+
Step 43860/50462 | Loss: 2.4531 | LR: 2.14e-05 | 1917.6ms/step | 31.3 samples/s
|
| 438 |
+
Step 43870/50462 | Loss: 2.4381 | LR: 2.14e-05 | 1919.1ms/step | 31.3 samples/s
|
| 439 |
+
Step 43880/50462 | Loss: 2.4783 | LR: 2.13e-05 | 1920.1ms/step | 31.2 samples/s
|
| 440 |
+
Step 43890/50462 | Loss: 2.4974 | LR: 2.13e-05 | 1919.0ms/step | 31.3 samples/s
|
| 441 |
+
Step 43900/50462 | Loss: 2.4290 | LR: 2.12e-05 | 1922.2ms/step | 31.2 samples/s
|
| 442 |
+
Step 43910/50462 | Loss: 2.4281 | LR: 2.11e-05 | 1924.4ms/step | 31.2 samples/s
|
| 443 |
+
Step 43920/50462 | Loss: 2.4434 | LR: 2.11e-05 | 1923.0ms/step | 31.2 samples/s
|
| 444 |
+
Step 43930/50462 | Loss: 2.4642 | LR: 2.10e-05 | 1922.2ms/step | 31.2 samples/s
|
| 445 |
+
Step 43940/50462 | Loss: 2.4623 | LR: 2.09e-05 | 1923.2ms/step | 31.2 samples/s
|
| 446 |
+
Step 43950/50462 | Loss: 2.4378 | LR: 2.09e-05 | 1924.2ms/step | 31.2 samples/s
|
| 447 |
+
Step 43960/50462 | Loss: 2.4489 | LR: 2.08e-05 | 1925.6ms/step | 31.2 samples/s
|
| 448 |
+
Step 43970/50462 | Loss: 2.4478 | LR: 2.07e-05 | 1925.0ms/step | 31.2 samples/s
|
| 449 |
+
Step 43980/50462 | Loss: 2.4438 | LR: 2.07e-05 | 1922.3ms/step | 31.2 samples/s
|
| 450 |
+
Step 43990/50462 | Loss: 2.4061 | LR: 2.06e-05 | 1924.1ms/step | 31.2 samples/s
|
| 451 |
+
Step 44000/50462 | Loss: 2.4359 | LR: 2.06e-05 | 1922.2ms/step | 31.2 samples/s
|
| 452 |
+
Validation at step 44000: Loss = 2.4995, Perplexity = 12.18
|
| 453 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44000
|
| 454 |
+
Step 44010/50462 | Loss: 2.4252 | LR: 2.05e-05 | 3301.3ms/step | 18.2 samples/s
|
| 455 |
+
Step 44020/50462 | Loss: 2.4414 | LR: 2.04e-05 | 1915.6ms/step | 31.3 samples/s
|
| 456 |
+
Step 44030/50462 | Loss: 2.4204 | LR: 2.04e-05 | 1917.8ms/step | 31.3 samples/s
|
| 457 |
+
Step 44040/50462 | Loss: 2.4698 | LR: 2.03e-05 | 1916.9ms/step | 31.3 samples/s
|
| 458 |
+
Step 44050/50462 | Loss: 2.4514 | LR: 2.02e-05 | 1917.0ms/step | 31.3 samples/s
|
| 459 |
+
Step 44060/50462 | Loss: 2.4378 | LR: 2.02e-05 | 1919.8ms/step | 31.3 samples/s
|
| 460 |
+
Step 44070/50462 | Loss: 2.4463 | LR: 2.01e-05 | 1921.9ms/step | 31.2 samples/s
|
| 461 |
+
Step 44080/50462 | Loss: 2.4620 | LR: 2.01e-05 | 1918.5ms/step | 31.3 samples/s
|
| 462 |
+
Step 44090/50462 | Loss: 2.4916 | LR: 2.00e-05 | 1922.8ms/step | 31.2 samples/s
|
| 463 |
+
Step 44100/50462 | Loss: 2.4125 | LR: 1.99e-05 | 1919.2ms/step | 31.3 samples/s
|
| 464 |
+
Step 44110/50462 | Loss: 2.4522 | LR: 1.99e-05 | 1920.2ms/step | 31.2 samples/s
|
| 465 |
+
Step 44120/50462 | Loss: 2.4452 | LR: 1.98e-05 | 1917.5ms/step | 31.3 samples/s
|
| 466 |
+
Step 44130/50462 | Loss: 2.4243 | LR: 1.97e-05 | 1914.4ms/step | 31.3 samples/s
|
| 467 |
+
Step 44140/50462 | Loss: 2.4771 | LR: 1.97e-05 | 1918.5ms/step | 31.3 samples/s
|
| 468 |
+
Step 44150/50462 | Loss: 2.4300 | LR: 1.96e-05 | 1919.2ms/step | 31.3 samples/s
|
| 469 |
+
Step 44160/50462 | Loss: 2.4304 | LR: 1.96e-05 | 1918.8ms/step | 31.3 samples/s
|
| 470 |
+
Step 44170/50462 | Loss: 2.4386 | LR: 1.95e-05 | 1918.7ms/step | 31.3 samples/s
|
| 471 |
+
Step 44180/50462 | Loss: 2.4345 | LR: 1.94e-05 | 1919.7ms/step | 31.3 samples/s
|
| 472 |
+
Step 44190/50462 | Loss: 2.4743 | LR: 1.94e-05 | 1917.6ms/step | 31.3 samples/s
|
| 473 |
+
Step 44200/50462 | Loss: 2.4926 | LR: 1.93e-05 | 1917.8ms/step | 31.3 samples/s
|
| 474 |
+
Step 44210/50462 | Loss: 2.4853 | LR: 1.93e-05 | 1919.5ms/step | 31.3 samples/s
|
| 475 |
+
Step 44220/50462 | Loss: 2.4625 | LR: 1.92e-05 | 1920.7ms/step | 31.2 samples/s
|
| 476 |
+
Step 44230/50462 | Loss: 2.4772 | LR: 1.91e-05 | 1918.9ms/step | 31.3 samples/s
|
| 477 |
+
Step 44240/50462 | Loss: 2.5026 | LR: 1.91e-05 | 1919.2ms/step | 31.3 samples/s
|
| 478 |
+
Step 44250/50462 | Loss: 2.4596 | LR: 1.90e-05 | 1919.5ms/step | 31.3 samples/s
|
| 479 |
+
Validation at step 44250: Loss = 2.4990, Perplexity = 12.17
|
| 480 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44250
|
| 481 |
+
Step 44260/50462 | Loss: 2.4803 | LR: 1.90e-05 | 3251.3ms/step | 18.5 samples/s
|
| 482 |
+
Step 44270/50462 | Loss: 2.4711 | LR: 1.89e-05 | 1916.0ms/step | 31.3 samples/s
|
| 483 |
+
Step 44280/50462 | Loss: 2.4804 | LR: 1.88e-05 | 1918.4ms/step | 31.3 samples/s
|
| 484 |
+
Step 44290/50462 | Loss: 2.4909 | LR: 1.88e-05 | 1919.7ms/step | 31.3 samples/s
|
| 485 |
+
Step 44300/50462 | Loss: 2.4937 | LR: 1.87e-05 | 1919.4ms/step | 31.3 samples/s
|
| 486 |
+
Step 44310/50462 | Loss: 2.4347 | LR: 1.87e-05 | 1923.9ms/step | 31.2 samples/s
|
| 487 |
+
Step 44320/50462 | Loss: 2.4798 | LR: 1.86e-05 | 1924.2ms/step | 31.2 samples/s
|
| 488 |
+
Step 44330/50462 | Loss: 2.4634 | LR: 1.85e-05 | 1920.9ms/step | 31.2 samples/s
|
| 489 |
+
Step 44340/50462 | Loss: 2.4542 | LR: 1.85e-05 | 1923.5ms/step | 31.2 samples/s
|
| 490 |
+
Step 44350/50462 | Loss: 2.4359 | LR: 1.84e-05 | 1921.9ms/step | 31.2 samples/s
|
| 491 |
+
Step 44360/50462 | Loss: 2.4608 | LR: 1.84e-05 | 1923.0ms/step | 31.2 samples/s
|
| 492 |
+
Step 44370/50462 | Loss: 2.4880 | LR: 1.83e-05 | 1923.1ms/step | 31.2 samples/s
|
| 493 |
+
Step 44380/50462 | Loss: 2.4324 | LR: 1.82e-05 | 1925.8ms/step | 31.2 samples/s
|
| 494 |
+
Step 44390/50462 | Loss: 2.4410 | LR: 1.82e-05 | 1925.0ms/step | 31.2 samples/s
|
| 495 |
+
Step 44400/50462 | Loss: 2.4964 | LR: 1.81e-05 | 1925.9ms/step | 31.2 samples/s
|
| 496 |
+
Step 44410/50462 | Loss: 2.4777 | LR: 1.81e-05 | 1924.4ms/step | 31.2 samples/s
|
| 497 |
+
Step 44420/50462 | Loss: 2.4611 | LR: 1.80e-05 | 1922.7ms/step | 31.2 samples/s
|
| 498 |
+
Step 44430/50462 | Loss: 2.4499 | LR: 1.79e-05 | 1927.5ms/step | 31.1 samples/s
|
| 499 |
+
Step 44440/50462 | Loss: 2.4295 | LR: 1.79e-05 | 1926.6ms/step | 31.1 samples/s
|
| 500 |
+
Step 44450/50462 | Loss: 2.4659 | LR: 1.78e-05 | 1922.5ms/step | 31.2 samples/s
|
| 501 |
+
Step 44460/50462 | Loss: 2.5139 | LR: 1.78e-05 | 1920.8ms/step | 31.2 samples/s
|
| 502 |
+
Step 44470/50462 | Loss: 2.4357 | LR: 1.77e-05 | 1925.5ms/step | 31.2 samples/s
|
| 503 |
+
Step 44480/50462 | Loss: 2.4974 | LR: 1.77e-05 | 1922.4ms/step | 31.2 samples/s
|
| 504 |
+
Step 44490/50462 | Loss: 2.4779 | LR: 1.76e-05 | 1922.2ms/step | 31.2 samples/s
|
| 505 |
+
Step 44500/50462 | Loss: 2.4207 | LR: 1.75e-05 | 1923.1ms/step | 31.2 samples/s
|
| 506 |
+
Validation at step 44500: Loss = 2.4989, Perplexity = 12.17
|
| 507 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44500
|
| 508 |
+
Step 44510/50462 | Loss: 2.4830 | LR: 1.75e-05 | 3274.3ms/step | 18.3 samples/s
|
| 509 |
+
Step 44520/50462 | Loss: 2.4782 | LR: 1.74e-05 | 1918.0ms/step | 31.3 samples/s
|
| 510 |
+
Step 44530/50462 | Loss: 2.4588 | LR: 1.74e-05 | 1921.2ms/step | 31.2 samples/s
|
| 511 |
+
Step 44540/50462 | Loss: 2.4542 | LR: 1.73e-05 | 1918.9ms/step | 31.3 samples/s
|
| 512 |
+
Step 44550/50462 | Loss: 2.5151 | LR: 1.72e-05 | 1915.7ms/step | 31.3 samples/s
|
| 513 |
+
Step 44560/50462 | Loss: 2.4647 | LR: 1.72e-05 | 1916.2ms/step | 31.3 samples/s
|
| 514 |
+
Step 44570/50462 | Loss: 2.4655 | LR: 1.71e-05 | 1917.4ms/step | 31.3 samples/s
|
| 515 |
+
Step 44580/50462 | Loss: 2.4762 | LR: 1.71e-05 | 1917.9ms/step | 31.3 samples/s
|
| 516 |
+
Step 44590/50462 | Loss: 2.4583 | LR: 1.70e-05 | 1918.5ms/step | 31.3 samples/s
|
| 517 |
+
Step 44600/50462 | Loss: 2.4609 | LR: 1.70e-05 | 1915.5ms/step | 31.3 samples/s
|
| 518 |
+
Step 44610/50462 | Loss: 2.4555 | LR: 1.69e-05 | 1917.9ms/step | 31.3 samples/s
|
| 519 |
+
Step 44620/50462 | Loss: 2.4735 | LR: 1.68e-05 | 1916.6ms/step | 31.3 samples/s
|
| 520 |
+
Step 44630/50462 | Loss: 2.4707 | LR: 1.68e-05 | 1918.0ms/step | 31.3 samples/s
|
| 521 |
+
Step 44640/50462 | Loss: 2.4972 | LR: 1.67e-05 | 1918.0ms/step | 31.3 samples/s
|
| 522 |
+
Step 44650/50462 | Loss: 2.4945 | LR: 1.67e-05 | 1917.8ms/step | 31.3 samples/s
|
| 523 |
+
Step 44660/50462 | Loss: 2.5098 | LR: 1.66e-05 | 1918.4ms/step | 31.3 samples/s
|
| 524 |
+
Step 44670/50462 | Loss: 2.4512 | LR: 1.66e-05 | 1918.7ms/step | 31.3 samples/s
|
| 525 |
+
Step 44680/50462 | Loss: 2.4673 | LR: 1.65e-05 | 1919.0ms/step | 31.3 samples/s
|
| 526 |
+
Step 44690/50462 | Loss: 2.4847 | LR: 1.64e-05 | 1917.0ms/step | 31.3 samples/s
|
| 527 |
+
Step 44700/50462 | Loss: 2.4670 | LR: 1.64e-05 | 1917.0ms/step | 31.3 samples/s
|
| 528 |
+
Step 44710/50462 | Loss: 2.4709 | LR: 1.63e-05 | 1915.9ms/step | 31.3 samples/s
|
| 529 |
+
Step 44720/50462 | Loss: 2.4967 | LR: 1.63e-05 | 1919.8ms/step | 31.3 samples/s
|
| 530 |
+
Step 44730/50462 | Loss: 2.4918 | LR: 1.62e-05 | 1916.4ms/step | 31.3 samples/s
|
| 531 |
+
Step 44740/50462 | Loss: 2.4737 | LR: 1.62e-05 | 1917.5ms/step | 31.3 samples/s
|
| 532 |
+
Step 44750/50462 | Loss: 2.4531 | LR: 1.61e-05 | 1917.3ms/step | 31.3 samples/s
|
| 533 |
+
Validation at step 44750: Loss = 2.4989, Perplexity = 12.17
|
| 534 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-44750
|
| 535 |
+
Step 44760/50462 | Loss: 2.4520 | LR: 1.61e-05 | 3258.3ms/step | 18.4 samples/s
|
| 536 |
+
Step 44770/50462 | Loss: 2.4371 | LR: 1.60e-05 | 1918.1ms/step | 31.3 samples/s
|
| 537 |
+
Step 44780/50462 | Loss: 2.4705 | LR: 1.59e-05 | 1919.3ms/step | 31.3 samples/s
|
| 538 |
+
Step 44790/50462 | Loss: 2.5051 | LR: 1.59e-05 | 1920.8ms/step | 31.2 samples/s
|
| 539 |
+
Step 44800/50462 | Loss: 2.4233 | LR: 1.58e-05 | 1923.4ms/step | 31.2 samples/s
|
| 540 |
+
Step 44810/50462 | Loss: 2.4711 | LR: 1.58e-05 | 1921.8ms/step | 31.2 samples/s
|
| 541 |
+
Step 44820/50462 | Loss: 2.4165 | LR: 1.57e-05 | 1920.4ms/step | 31.2 samples/s
|
| 542 |
+
Step 44830/50462 | Loss: 2.4607 | LR: 1.57e-05 | 1926.1ms/step | 31.2 samples/s
|
| 543 |
+
Step 44840/50462 | Loss: 2.4703 | LR: 1.56e-05 | 1924.7ms/step | 31.2 samples/s
|
| 544 |
+
Step 44850/50462 | Loss: 2.4942 | LR: 1.56e-05 | 1924.5ms/step | 31.2 samples/s
|
| 545 |
+
Step 44860/50462 | Loss: 2.4232 | LR: 1.55e-05 | 1921.1ms/step | 31.2 samples/s
|
| 546 |
+
Step 44870/50462 | Loss: 2.4886 | LR: 1.54e-05 | 1922.1ms/step | 31.2 samples/s
|
| 547 |
+
Step 44880/50462 | Loss: 2.4622 | LR: 1.54e-05 | 1921.8ms/step | 31.2 samples/s
|
| 548 |
+
Step 44890/50462 | Loss: 2.4408 | LR: 1.53e-05 | 1921.2ms/step | 31.2 samples/s
|
| 549 |
+
Step 44900/50462 | Loss: 2.4750 | LR: 1.53e-05 | 1925.0ms/step | 31.2 samples/s
|
| 550 |
+
Step 44910/50462 | Loss: 2.4820 | LR: 1.52e-05 | 1921.9ms/step | 31.2 samples/s
|
| 551 |
+
Step 44920/50462 | Loss: 2.4474 | LR: 1.52e-05 | 1918.8ms/step | 31.3 samples/s
|
| 552 |
+
Step 44930/50462 | Loss: 2.4809 | LR: 1.51e-05 | 1923.2ms/step | 31.2 samples/s
|
| 553 |
+
Step 44940/50462 | Loss: 2.4949 | LR: 1.51e-05 | 1927.2ms/step | 31.1 samples/s
|
| 554 |
+
Step 44950/50462 | Loss: 2.4415 | LR: 1.50e-05 | 1923.8ms/step | 31.2 samples/s
|
| 555 |
+
Step 44960/50462 | Loss: 2.4358 | LR: 1.50e-05 | 1923.1ms/step | 31.2 samples/s
|
| 556 |
+
Step 44970/50462 | Loss: 2.4449 | LR: 1.49e-05 | 1921.4ms/step | 31.2 samples/s
|
| 557 |
+
Step 44980/50462 | Loss: 2.4818 | LR: 1.49e-05 | 1922.1ms/step | 31.2 samples/s
|
| 558 |
+
Step 44990/50462 | Loss: 2.4448 | LR: 1.48e-05 | 1925.0ms/step | 31.2 samples/s
|
| 559 |
+
Step 45000/50462 | Loss: 2.4615 | LR: 1.47e-05 | 1922.8ms/step | 31.2 samples/s
|
| 560 |
+
Validation at step 45000: Loss = 2.4988, Perplexity = 12.17
|
| 561 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45000
|
| 562 |
+
Step 45010/50462 | Loss: 2.4564 | LR: 1.47e-05 | 3297.9ms/step | 18.2 samples/s
|
| 563 |
+
Step 45020/50462 | Loss: 2.4558 | LR: 1.46e-05 | 1913.5ms/step | 31.4 samples/s
|
| 564 |
+
Step 45030/50462 | Loss: 2.4511 | LR: 1.46e-05 | 1921.1ms/step | 31.2 samples/s
|
| 565 |
+
Step 45040/50462 | Loss: 2.4349 | LR: 1.45e-05 | 1919.2ms/step | 31.3 samples/s
|
| 566 |
+
Step 45050/50462 | Loss: 2.4490 | LR: 1.45e-05 | 1921.6ms/step | 31.2 samples/s
|
| 567 |
+
Step 45060/50462 | Loss: 2.4522 | LR: 1.44e-05 | 1921.0ms/step | 31.2 samples/s
|
| 568 |
+
Step 45070/50462 | Loss: 2.4234 | LR: 1.44e-05 | 1922.6ms/step | 31.2 samples/s
|
| 569 |
+
Step 45080/50462 | Loss: 2.4709 | LR: 1.43e-05 | 1925.0ms/step | 31.2 samples/s
|
| 570 |
+
Step 45090/50462 | Loss: 2.4800 | LR: 1.43e-05 | 1922.0ms/step | 31.2 samples/s
|
| 571 |
+
Step 45100/50462 | Loss: 2.4691 | LR: 1.42e-05 | 1924.3ms/step | 31.2 samples/s
|
| 572 |
+
Step 45110/50462 | Loss: 2.4342 | LR: 1.42e-05 | 1921.2ms/step | 31.2 samples/s
|
| 573 |
+
Step 45120/50462 | Loss: 2.4736 | LR: 1.41e-05 | 1925.0ms/step | 31.2 samples/s
|
| 574 |
+
Step 45130/50462 | Loss: 2.4437 | LR: 1.41e-05 | 1926.2ms/step | 31.1 samples/s
|
| 575 |
+
Step 45140/50462 | Loss: 2.5367 | LR: 1.40e-05 | 1925.4ms/step | 31.2 samples/s
|
| 576 |
+
Step 45150/50462 | Loss: 2.4483 | LR: 1.40e-05 | 1925.4ms/step | 31.2 samples/s
|
| 577 |
+
Step 45160/50462 | Loss: 2.4879 | LR: 1.39e-05 | 1923.7ms/step | 31.2 samples/s
|
| 578 |
+
Step 45170/50462 | Loss: 2.4924 | LR: 1.39e-05 | 1925.0ms/step | 31.2 samples/s
|
| 579 |
+
Step 45180/50462 | Loss: 2.4560 | LR: 1.38e-05 | 1925.9ms/step | 31.2 samples/s
|
| 580 |
+
Step 45190/50462 | Loss: 2.4832 | LR: 1.37e-05 | 1918.1ms/step | 31.3 samples/s
|
| 581 |
+
Step 45200/50462 | Loss: 2.4594 | LR: 1.37e-05 | 1919.9ms/step | 31.3 samples/s
|
| 582 |
+
Step 45210/50462 | Loss: 2.4270 | LR: 1.36e-05 | 1918.0ms/step | 31.3 samples/s
|
| 583 |
+
Step 45220/50462 | Loss: 2.4418 | LR: 1.36e-05 | 1921.2ms/step | 31.2 samples/s
|
| 584 |
+
Step 45230/50462 | Loss: 2.5007 | LR: 1.35e-05 | 1916.5ms/step | 31.3 samples/s
|
| 585 |
+
Step 45240/50462 | Loss: 2.5106 | LR: 1.35e-05 | 1919.5ms/step | 31.3 samples/s
|
| 586 |
+
Step 45250/50462 | Loss: 2.4533 | LR: 1.34e-05 | 1916.5ms/step | 31.3 samples/s
|
| 587 |
+
Validation at step 45250: Loss = 2.4988, Perplexity = 12.17
|
| 588 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45250
|
| 589 |
+
Step 45260/50462 | Loss: 2.4836 | LR: 1.34e-05 | 3259.1ms/step | 18.4 samples/s
|
| 590 |
+
Step 45270/50462 | Loss: 2.5005 | LR: 1.33e-05 | 1914.5ms/step | 31.3 samples/s
|
| 591 |
+
Step 45280/50462 | Loss: 2.4417 | LR: 1.33e-05 | 1919.6ms/step | 31.3 samples/s
|
| 592 |
+
Step 45290/50462 | Loss: 2.4753 | LR: 1.32e-05 | 1920.2ms/step | 31.2 samples/s
|
| 593 |
+
Step 45300/50462 | Loss: 2.5373 | LR: 1.32e-05 | 1921.4ms/step | 31.2 samples/s
|
| 594 |
+
Step 45310/50462 | Loss: 2.4478 | LR: 1.31e-05 | 1921.8ms/step | 31.2 samples/s
|
| 595 |
+
Step 45320/50462 | Loss: 2.4725 | LR: 1.31e-05 | 1921.9ms/step | 31.2 samples/s
|
| 596 |
+
Step 45330/50462 | Loss: 2.4741 | LR: 1.30e-05 | 1924.7ms/step | 31.2 samples/s
|
| 597 |
+
Step 45340/50462 | Loss: 2.4498 | LR: 1.30e-05 | 1922.6ms/step | 31.2 samples/s
|
| 598 |
+
Step 45350/50462 | Loss: 2.4930 | LR: 1.29e-05 | 1923.5ms/step | 31.2 samples/s
|
| 599 |
+
Step 45360/50462 | Loss: 2.4648 | LR: 1.29e-05 | 1921.3ms/step | 31.2 samples/s
|
| 600 |
+
Step 45370/50462 | Loss: 2.4564 | LR: 1.28e-05 | 1921.6ms/step | 31.2 samples/s
|
| 601 |
+
Step 45380/50462 | Loss: 2.4427 | LR: 1.28e-05 | 1925.5ms/step | 31.2 samples/s
|
| 602 |
+
Step 45390/50462 | Loss: 2.4447 | LR: 1.27e-05 | 1923.7ms/step | 31.2 samples/s
|
| 603 |
+
Step 45400/50462 | Loss: 2.4423 | LR: 1.27e-05 | 1922.4ms/step | 31.2 samples/s
|
| 604 |
+
Step 45410/50462 | Loss: 2.4823 | LR: 1.26e-05 | 1925.1ms/step | 31.2 samples/s
|
| 605 |
+
Step 45420/50462 | Loss: 2.4698 | LR: 1.26e-05 | 1924.6ms/step | 31.2 samples/s
|
| 606 |
+
Step 45430/50462 | Loss: 2.5209 | LR: 1.25e-05 | 1923.3ms/step | 31.2 samples/s
|
| 607 |
+
Step 45440/50462 | Loss: 2.4659 | LR: 1.25e-05 | 1922.7ms/step | 31.2 samples/s
|
| 608 |
+
Step 45450/50462 | Loss: 2.5022 | LR: 1.24e-05 | 1927.5ms/step | 31.1 samples/s
|
| 609 |
+
Step 45460/50462 | Loss: 2.4470 | LR: 1.24e-05 | 1928.4ms/step | 31.1 samples/s
|
| 610 |
+
Step 45470/50462 | Loss: 2.4433 | LR: 1.23e-05 | 1925.4ms/step | 31.2 samples/s
|
| 611 |
+
Step 45480/50462 | Loss: 2.4502 | LR: 1.23e-05 | 1924.0ms/step | 31.2 samples/s
|
| 612 |
+
Step 45490/50462 | Loss: 2.4775 | LR: 1.22e-05 | 1923.6ms/step | 31.2 samples/s
|
| 613 |
+
Step 45500/50462 | Loss: 2.4669 | LR: 1.22e-05 | 1925.9ms/step | 31.2 samples/s
|
| 614 |
+
Validation at step 45500: Loss = 2.4988, Perplexity = 12.17
|
| 615 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45500
|
| 616 |
+
Step 45510/50462 | Loss: 2.4472 | LR: 1.21e-05 | 3256.6ms/step | 18.4 samples/s
|
| 617 |
+
Step 45520/50462 | Loss: 2.4466 | LR: 1.21e-05 | 1918.1ms/step | 31.3 samples/s
|
| 618 |
+
Step 45530/50462 | Loss: 2.4601 | LR: 1.20e-05 | 1918.6ms/step | 31.3 samples/s
|
| 619 |
+
Step 45540/50462 | Loss: 2.4692 | LR: 1.20e-05 | 1921.5ms/step | 31.2 samples/s
|
| 620 |
+
Step 45550/50462 | Loss: 2.4633 | LR: 1.19e-05 | 1922.5ms/step | 31.2 samples/s
|
| 621 |
+
Step 45560/50462 | Loss: 2.4711 | LR: 1.19e-05 | 1925.7ms/step | 31.2 samples/s
|
| 622 |
+
Step 45570/50462 | Loss: 2.4809 | LR: 1.19e-05 | 1922.6ms/step | 31.2 samples/s
|
| 623 |
+
Step 45580/50462 | Loss: 2.4433 | LR: 1.18e-05 | 1923.5ms/step | 31.2 samples/s
|
| 624 |
+
Step 45590/50462 | Loss: 2.4561 | LR: 1.18e-05 | 1921.5ms/step | 31.2 samples/s
|
| 625 |
+
Step 45600/50462 | Loss: 2.5045 | LR: 1.17e-05 | 1922.3ms/step | 31.2 samples/s
|
| 626 |
+
Step 45610/50462 | Loss: 2.4789 | LR: 1.17e-05 | 1917.8ms/step | 31.3 samples/s
|
| 627 |
+
Step 45620/50462 | Loss: 2.4775 | LR: 1.16e-05 | 1917.5ms/step | 31.3 samples/s
|
| 628 |
+
Step 45630/50462 | Loss: 2.4762 | LR: 1.16e-05 | 1920.6ms/step | 31.2 samples/s
|
| 629 |
+
Step 45640/50462 | Loss: 2.5028 | LR: 1.15e-05 | 1919.1ms/step | 31.3 samples/s
|
| 630 |
+
Step 45650/50462 | Loss: 2.4313 | LR: 1.15e-05 | 1919.1ms/step | 31.3 samples/s
|
| 631 |
+
Step 45660/50462 | Loss: 2.4793 | LR: 1.14e-05 | 1919.2ms/step | 31.3 samples/s
|
| 632 |
+
Step 45670/50462 | Loss: 2.4838 | LR: 1.14e-05 | 1918.9ms/step | 31.3 samples/s
|
| 633 |
+
Step 45680/50462 | Loss: 2.4650 | LR: 1.13e-05 | 1914.8ms/step | 31.3 samples/s
|
| 634 |
+
Step 45690/50462 | Loss: 2.4721 | LR: 1.13e-05 | 1918.7ms/step | 31.3 samples/s
|
| 635 |
+
Step 45700/50462 | Loss: 2.5494 | LR: 1.12e-05 | 1919.1ms/step | 31.3 samples/s
|
| 636 |
+
Step 45710/50462 | Loss: 2.4502 | LR: 1.12e-05 | 1917.5ms/step | 31.3 samples/s
|
| 637 |
+
Step 45720/50462 | Loss: 2.4356 | LR: 1.11e-05 | 1920.6ms/step | 31.2 samples/s
|
| 638 |
+
Step 45730/50462 | Loss: 2.4605 | LR: 1.11e-05 | 1918.5ms/step | 31.3 samples/s
|
| 639 |
+
Step 45740/50462 | Loss: 2.4800 | LR: 1.10e-05 | 1917.8ms/step | 31.3 samples/s
|
| 640 |
+
Step 45750/50462 | Loss: 2.4708 | LR: 1.10e-05 | 1917.3ms/step | 31.3 samples/s
|
| 641 |
+
Validation at step 45750: Loss = 2.4988, Perplexity = 12.17
|
| 642 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-45750
|
| 643 |
+
Step 45760/50462 | Loss: 2.4833 | LR: 1.10e-05 | 3266.3ms/step | 18.4 samples/s
|
| 644 |
+
Step 45770/50462 | Loss: 2.4648 | LR: 1.09e-05 | 1917.2ms/step | 31.3 samples/s
|
| 645 |
+
Step 45780/50462 | Loss: 2.4417 | LR: 1.09e-05 | 1917.4ms/step | 31.3 samples/s
|
| 646 |
+
Step 45790/50462 | Loss: 2.4261 | LR: 1.08e-05 | 1916.5ms/step | 31.3 samples/s
|
| 647 |
+
Step 45800/50462 | Loss: 2.5306 | LR: 1.08e-05 | 1918.6ms/step | 31.3 samples/s
|
| 648 |
+
Step 45810/50462 | Loss: 2.4634 | LR: 1.07e-05 | 1917.1ms/step | 31.3 samples/s
|
| 649 |
+
Step 45820/50462 | Loss: 2.4529 | LR: 1.07e-05 | 1919.8ms/step | 31.3 samples/s
|
| 650 |
+
Step 45830/50462 | Loss: 2.4374 | LR: 1.06e-05 | 1919.7ms/step | 31.3 samples/s
|
| 651 |
+
Step 45840/50462 | Loss: 2.4661 | LR: 1.06e-05 | 1916.9ms/step | 31.3 samples/s
|
| 652 |
+
Step 45850/50462 | Loss: 2.4758 | LR: 1.05e-05 | 1916.7ms/step | 31.3 samples/s
|
| 653 |
+
Step 45860/50462 | Loss: 2.4522 | LR: 1.05e-05 | 1919.5ms/step | 31.3 samples/s
|
| 654 |
+
Step 45870/50462 | Loss: 2.4374 | LR: 1.05e-05 | 1919.9ms/step | 31.3 samples/s
|
| 655 |
+
Step 45880/50462 | Loss: 2.5036 | LR: 1.04e-05 | 1918.5ms/step | 31.3 samples/s
|
| 656 |
+
Step 45890/50462 | Loss: 2.4574 | LR: 1.04e-05 | 1919.1ms/step | 31.3 samples/s
|
| 657 |
+
Step 45900/50462 | Loss: 2.4612 | LR: 1.03e-05 | 1921.5ms/step | 31.2 samples/s
|
| 658 |
+
Step 45910/50462 | Loss: 2.4739 | LR: 1.03e-05 | 1919.9ms/step | 31.3 samples/s
|
| 659 |
+
Step 45920/50462 | Loss: 2.4628 | LR: 1.02e-05 | 1917.6ms/step | 31.3 samples/s
|
| 660 |
+
Step 45930/50462 | Loss: 2.5061 | LR: 1.02e-05 | 1919.6ms/step | 31.3 samples/s
|
| 661 |
+
Step 45940/50462 | Loss: 2.4795 | LR: 1.01e-05 | 1920.5ms/step | 31.2 samples/s
|
| 662 |
+
Step 45950/50462 | Loss: 2.4315 | LR: 1.01e-05 | 1919.0ms/step | 31.3 samples/s
|
| 663 |
+
Step 45960/50462 | Loss: 2.4746 | LR: 1.01e-05 | 1918.9ms/step | 31.3 samples/s
|
| 664 |
+
Step 45970/50462 | Loss: 2.4716 | LR: 1.00e-05 | 1917.6ms/step | 31.3 samples/s
|
| 665 |
+
Step 45980/50462 | Loss: 2.4656 | LR: 9.96e-06 | 1918.6ms/step | 31.3 samples/s
|
| 666 |
+
Step 45990/50462 | Loss: 2.4984 | LR: 9.92e-06 | 1919.3ms/step | 31.3 samples/s
|
| 667 |
+
Step 46000/50462 | Loss: 2.4757 | LR: 9.87e-06 | 1919.4ms/step | 31.3 samples/s
|
| 668 |
+
Validation at step 46000: Loss = 2.4987, Perplexity = 12.17
|
| 669 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46000
|
| 670 |
+
Step 46010/50462 | Loss: 2.4585 | LR: 9.83e-06 | 3253.9ms/step | 18.4 samples/s
|
| 671 |
+
Step 46020/50462 | Loss: 2.4217 | LR: 9.79e-06 | 1920.0ms/step | 31.3 samples/s
|
| 672 |
+
Step 46030/50462 | Loss: 2.4556 | LR: 9.74e-06 | 1921.3ms/step | 31.2 samples/s
|
| 673 |
+
Step 46040/50462 | Loss: 2.4790 | LR: 9.70e-06 | 1924.0ms/step | 31.2 samples/s
|
| 674 |
+
Step 46050/50462 | Loss: 2.4203 | LR: 9.65e-06 | 1924.2ms/step | 31.2 samples/s
|
| 675 |
+
Step 46060/50462 | Loss: 2.4171 | LR: 9.61e-06 | 1925.3ms/step | 31.2 samples/s
|
| 676 |
+
Step 46070/50462 | Loss: 2.4707 | LR: 9.57e-06 | 1922.5ms/step | 31.2 samples/s
|
| 677 |
+
Step 46080/50462 | Loss: 2.4632 | LR: 9.52e-06 | 1920.8ms/step | 31.2 samples/s
|
| 678 |
+
Step 46090/50462 | Loss: 2.4981 | LR: 9.48e-06 | 1926.1ms/step | 31.2 samples/s
|
| 679 |
+
Step 46100/50462 | Loss: 2.5062 | LR: 9.44e-06 | 1923.3ms/step | 31.2 samples/s
|
| 680 |
+
Step 46110/50462 | Loss: 2.4697 | LR: 9.40e-06 | 1925.7ms/step | 31.2 samples/s
|
| 681 |
+
Step 46120/50462 | Loss: 2.4609 | LR: 9.35e-06 | 1921.6ms/step | 31.2 samples/s
|
| 682 |
+
Step 46130/50462 | Loss: 2.4842 | LR: 9.31e-06 | 1923.0ms/step | 31.2 samples/s
|
| 683 |
+
Step 46140/50462 | Loss: 2.5019 | LR: 9.27e-06 | 1924.3ms/step | 31.2 samples/s
|
| 684 |
+
Step 46150/50462 | Loss: 2.4675 | LR: 9.22e-06 | 1922.2ms/step | 31.2 samples/s
|
| 685 |
+
Step 46160/50462 | Loss: 2.4752 | LR: 9.18e-06 | 1924.1ms/step | 31.2 samples/s
|
| 686 |
+
Step 46170/50462 | Loss: 2.4904 | LR: 9.14e-06 | 1921.1ms/step | 31.2 samples/s
|
| 687 |
+
Step 46180/50462 | Loss: 2.5061 | LR: 9.10e-06 | 1927.1ms/step | 31.1 samples/s
|
| 688 |
+
Step 46190/50462 | Loss: 2.4397 | LR: 9.06e-06 | 1921.3ms/step | 31.2 samples/s
|
| 689 |
+
Step 46200/50462 | Loss: 2.4606 | LR: 9.01e-06 | 1924.9ms/step | 31.2 samples/s
|
| 690 |
+
Step 46210/50462 | Loss: 2.4374 | LR: 8.97e-06 | 1923.3ms/step | 31.2 samples/s
|
| 691 |
+
Step 46220/50462 | Loss: 2.4603 | LR: 8.93e-06 | 1926.2ms/step | 31.1 samples/s
|
| 692 |
+
Step 46230/50462 | Loss: 2.4428 | LR: 8.89e-06 | 1924.9ms/step | 31.2 samples/s
|
| 693 |
+
Step 46240/50462 | Loss: 2.4603 | LR: 8.85e-06 | 1924.5ms/step | 31.2 samples/s
|
| 694 |
+
Step 46250/50462 | Loss: 2.4302 | LR: 8.80e-06 | 1927.6ms/step | 31.1 samples/s
|
| 695 |
+
Validation at step 46250: Loss = 2.4988, Perplexity = 12.17
|
| 696 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46250
|
| 697 |
+
Step 46260/50462 | Loss: 2.5131 | LR: 8.76e-06 | 3305.1ms/step | 18.2 samples/s
|
| 698 |
+
Step 46270/50462 | Loss: 2.4862 | LR: 8.72e-06 | 1918.8ms/step | 31.3 samples/s
|
| 699 |
+
Step 46280/50462 | Loss: 2.4912 | LR: 8.68e-06 | 1922.8ms/step | 31.2 samples/s
|
| 700 |
+
Step 46290/50462 | Loss: 2.4960 | LR: 8.64e-06 | 1923.7ms/step | 31.2 samples/s
|
| 701 |
+
Step 46300/50462 | Loss: 2.4733 | LR: 8.60e-06 | 1921.9ms/step | 31.2 samples/s
|
| 702 |
+
Step 46310/50462 | Loss: 2.4538 | LR: 8.56e-06 | 1922.6ms/step | 31.2 samples/s
|
| 703 |
+
Step 46320/50462 | Loss: 2.4543 | LR: 8.52e-06 | 1923.9ms/step | 31.2 samples/s
|
| 704 |
+
Step 46330/50462 | Loss: 2.4846 | LR: 8.47e-06 | 1923.6ms/step | 31.2 samples/s
|
| 705 |
+
Step 46340/50462 | Loss: 2.4799 | LR: 8.43e-06 | 1920.3ms/step | 31.2 samples/s
|
| 706 |
+
Step 46350/50462 | Loss: 2.4437 | LR: 8.39e-06 | 1922.9ms/step | 31.2 samples/s
|
| 707 |
+
Step 46360/50462 | Loss: 2.4783 | LR: 8.35e-06 | 1925.1ms/step | 31.2 samples/s
|
| 708 |
+
Step 46370/50462 | Loss: 2.4630 | LR: 8.31e-06 | 1928.3ms/step | 31.1 samples/s
|
| 709 |
+
Step 46380/50462 | Loss: 2.4529 | LR: 8.27e-06 | 1921.9ms/step | 31.2 samples/s
|
| 710 |
+
Step 46390/50462 | Loss: 2.4523 | LR: 8.23e-06 | 1924.4ms/step | 31.2 samples/s
|
| 711 |
+
Step 46400/50462 | Loss: 2.4968 | LR: 8.19e-06 | 1923.1ms/step | 31.2 samples/s
|
| 712 |
+
Step 46410/50462 | Loss: 2.4479 | LR: 8.15e-06 | 1924.4ms/step | 31.2 samples/s
|
| 713 |
+
Step 46420/50462 | Loss: 2.4419 | LR: 8.11e-06 | 1921.1ms/step | 31.2 samples/s
|
| 714 |
+
Step 46430/50462 | Loss: 2.4370 | LR: 8.07e-06 | 1924.7ms/step | 31.2 samples/s
|
| 715 |
+
Step 46440/50462 | Loss: 2.4812 | LR: 8.03e-06 | 1926.3ms/step | 31.1 samples/s
|
| 716 |
+
Step 46450/50462 | Loss: 2.4597 | LR: 7.99e-06 | 1922.6ms/step | 31.2 samples/s
|
| 717 |
+
Step 46460/50462 | Loss: 2.4850 | LR: 7.95e-06 | 1925.3ms/step | 31.2 samples/s
|
| 718 |
+
Step 46470/50462 | Loss: 2.4079 | LR: 7.91e-06 | 1924.7ms/step | 31.2 samples/s
|
| 719 |
+
Step 46480/50462 | Loss: 2.4684 | LR: 7.87e-06 | 1926.0ms/step | 31.2 samples/s
|
| 720 |
+
Step 46490/50462 | Loss: 2.4641 | LR: 7.83e-06 | 1925.4ms/step | 31.2 samples/s
|
| 721 |
+
Step 46500/50462 | Loss: 2.4763 | LR: 7.80e-06 | 1924.2ms/step | 31.2 samples/s
|
| 722 |
+
Validation at step 46500: Loss = 2.4987, Perplexity = 12.17
|
| 723 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46500
|
| 724 |
+
Step 46510/50462 | Loss: 2.4691 | LR: 7.76e-06 | 3379.2ms/step | 17.8 samples/s
|
| 725 |
+
Step 46520/50462 | Loss: 2.4238 | LR: 7.72e-06 | 1914.9ms/step | 31.3 samples/s
|
| 726 |
+
Step 46530/50462 | Loss: 2.4831 | LR: 7.68e-06 | 1919.3ms/step | 31.3 samples/s
|
| 727 |
+
Step 46540/50462 | Loss: 2.5015 | LR: 7.64e-06 | 1921.6ms/step | 31.2 samples/s
|
| 728 |
+
Step 46550/50462 | Loss: 2.4583 | LR: 7.60e-06 | 1924.9ms/step | 31.2 samples/s
|
| 729 |
+
Step 46560/50462 | Loss: 2.4948 | LR: 7.56e-06 | 1924.7ms/step | 31.2 samples/s
|
| 730 |
+
Step 46570/50462 | Loss: 2.4723 | LR: 7.52e-06 | 1923.2ms/step | 31.2 samples/s
|
| 731 |
+
Step 46580/50462 | Loss: 2.4477 | LR: 7.49e-06 | 1921.8ms/step | 31.2 samples/s
|
| 732 |
+
Step 46590/50462 | Loss: 2.4772 | LR: 7.45e-06 | 1919.9ms/step | 31.3 samples/s
|
| 733 |
+
Step 46600/50462 | Loss: 2.4728 | LR: 7.41e-06 | 1921.2ms/step | 31.2 samples/s
|
| 734 |
+
Step 46610/50462 | Loss: 2.4965 | LR: 7.37e-06 | 1916.2ms/step | 31.3 samples/s
|
| 735 |
+
Step 46620/50462 | Loss: 2.4331 | LR: 7.33e-06 | 1919.2ms/step | 31.3 samples/s
|
| 736 |
+
Step 46630/50462 | Loss: 2.4514 | LR: 7.29e-06 | 1915.5ms/step | 31.3 samples/s
|
| 737 |
+
Step 46640/50462 | Loss: 2.4242 | LR: 7.26e-06 | 1918.1ms/step | 31.3 samples/s
|
| 738 |
+
Step 46650/50462 | Loss: 2.5266 | LR: 7.22e-06 | 1919.0ms/step | 31.3 samples/s
|
| 739 |
+
Step 46660/50462 | Loss: 2.4518 | LR: 7.18e-06 | 1919.6ms/step | 31.3 samples/s
|
| 740 |
+
Step 46670/50462 | Loss: 2.4514 | LR: 7.14e-06 | 1918.6ms/step | 31.3 samples/s
|
| 741 |
+
Step 46680/50462 | Loss: 2.4667 | LR: 7.11e-06 | 1920.5ms/step | 31.2 samples/s
|
| 742 |
+
Step 46690/50462 | Loss: 2.4927 | LR: 7.07e-06 | 1921.2ms/step | 31.2 samples/s
|
| 743 |
+
Step 46700/50462 | Loss: 2.4737 | LR: 7.03e-06 | 1920.9ms/step | 31.2 samples/s
|
| 744 |
+
Step 46710/50462 | Loss: 2.4743 | LR: 6.99e-06 | 1917.6ms/step | 31.3 samples/s
|
| 745 |
+
Step 46720/50462 | Loss: 2.4817 | LR: 6.96e-06 | 1920.0ms/step | 31.3 samples/s
|
| 746 |
+
Step 46730/50462 | Loss: 2.4341 | LR: 6.92e-06 | 1917.6ms/step | 31.3 samples/s
|
| 747 |
+
Step 46740/50462 | Loss: 2.4589 | LR: 6.88e-06 | 1917.3ms/step | 31.3 samples/s
|
| 748 |
+
Step 46750/50462 | Loss: 2.3727 | LR: 6.85e-06 | 1919.4ms/step | 31.3 samples/s
|
| 749 |
+
Validation at step 46750: Loss = 2.4988, Perplexity = 12.17
|
| 750 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-46750
|
| 751 |
+
Step 46760/50462 | Loss: 2.4908 | LR: 6.81e-06 | 3259.7ms/step | 18.4 samples/s
|
| 752 |
+
Step 46770/50462 | Loss: 2.4642 | LR: 6.77e-06 | 1917.2ms/step | 31.3 samples/s
|
| 753 |
+
Step 46780/50462 | Loss: 2.4925 | LR: 6.74e-06 | 1919.8ms/step | 31.3 samples/s
|
| 754 |
+
Step 46790/50462 | Loss: 2.4438 | LR: 6.70e-06 | 1920.5ms/step | 31.2 samples/s
|
| 755 |
+
Step 46800/50462 | Loss: 2.4609 | LR: 6.66e-06 | 1916.0ms/step | 31.3 samples/s
|
| 756 |
+
Step 46810/50462 | Loss: 2.4237 | LR: 6.63e-06 | 1922.1ms/step | 31.2 samples/s
|
| 757 |
+
Step 46820/50462 | Loss: 2.4838 | LR: 6.59e-06 | 1921.4ms/step | 31.2 samples/s
|
| 758 |
+
Step 46830/50462 | Loss: 2.4396 | LR: 6.56e-06 | 1924.3ms/step | 31.2 samples/s
|
| 759 |
+
Step 46840/50462 | Loss: 2.4829 | LR: 6.52e-06 | 1924.1ms/step | 31.2 samples/s
|
| 760 |
+
Step 46850/50462 | Loss: 2.4540 | LR: 6.48e-06 | 1926.4ms/step | 31.1 samples/s
|
| 761 |
+
Step 46860/50462 | Loss: 2.4880 | LR: 6.45e-06 | 1925.9ms/step | 31.2 samples/s
|
| 762 |
+
Step 46870/50462 | Loss: 2.4926 | LR: 6.41e-06 | 1923.0ms/step | 31.2 samples/s
|
| 763 |
+
Step 46880/50462 | Loss: 2.4483 | LR: 6.38e-06 | 1924.0ms/step | 31.2 samples/s
|
| 764 |
+
Step 46890/50462 | Loss: 2.4671 | LR: 6.34e-06 | 1924.3ms/step | 31.2 samples/s
|
| 765 |
+
Step 46900/50462 | Loss: 2.4385 | LR: 6.31e-06 | 1925.6ms/step | 31.2 samples/s
|
| 766 |
+
Step 46910/50462 | Loss: 2.4853 | LR: 6.27e-06 | 1926.8ms/step | 31.1 samples/s
|
| 767 |
+
Step 46920/50462 | Loss: 2.4426 | LR: 6.24e-06 | 1924.7ms/step | 31.2 samples/s
|
| 768 |
+
Step 46930/50462 | Loss: 2.4577 | LR: 6.20e-06 | 1922.6ms/step | 31.2 samples/s
|
| 769 |
+
Step 46940/50462 | Loss: 2.4792 | LR: 6.17e-06 | 1925.1ms/step | 31.2 samples/s
|
| 770 |
+
Step 46950/50462 | Loss: 2.4722 | LR: 6.13e-06 | 1920.2ms/step | 31.2 samples/s
|
| 771 |
+
Step 46960/50462 | Loss: 2.4641 | LR: 6.10e-06 | 1925.4ms/step | 31.2 samples/s
|
| 772 |
+
Step 46970/50462 | Loss: 2.4542 | LR: 6.06e-06 | 1922.8ms/step | 31.2 samples/s
|
| 773 |
+
Step 46980/50462 | Loss: 2.4552 | LR: 6.03e-06 | 1925.5ms/step | 31.2 samples/s
|
| 774 |
+
Step 46990/50462 | Loss: 2.4703 | LR: 5.99e-06 | 1921.4ms/step | 31.2 samples/s
|
| 775 |
+
Step 47000/50462 | Loss: 2.4782 | LR: 5.96e-06 | 1926.4ms/step | 31.1 samples/s
|
| 776 |
+
Validation at step 47000: Loss = 2.4988, Perplexity = 12.17
|
| 777 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47000
|
| 778 |
+
Step 47010/50462 | Loss: 2.4535 | LR: 5.93e-06 | 3250.4ms/step | 18.5 samples/s
|
| 779 |
+
Step 47020/50462 | Loss: 2.4650 | LR: 5.89e-06 | 1917.8ms/step | 31.3 samples/s
|
| 780 |
+
Step 47030/50462 | Loss: 2.4534 | LR: 5.86e-06 | 1921.7ms/step | 31.2 samples/s
|
| 781 |
+
Step 47040/50462 | Loss: 2.4926 | LR: 5.82e-06 | 1916.4ms/step | 31.3 samples/s
|
| 782 |
+
Step 47050/50462 | Loss: 2.4695 | LR: 5.79e-06 | 1916.4ms/step | 31.3 samples/s
|
| 783 |
+
Step 47060/50462 | Loss: 2.4439 | LR: 5.76e-06 | 1917.8ms/step | 31.3 samples/s
|
| 784 |
+
Step 47070/50462 | Loss: 2.4558 | LR: 5.72e-06 | 1919.2ms/step | 31.3 samples/s
|
| 785 |
+
Step 47080/50462 | Loss: 2.4468 | LR: 5.69e-06 | 1917.5ms/step | 31.3 samples/s
|
| 786 |
+
Step 47090/50462 | Loss: 2.4588 | LR: 5.65e-06 | 1917.3ms/step | 31.3 samples/s
|
| 787 |
+
Step 47100/50462 | Loss: 2.4943 | LR: 5.62e-06 | 1919.1ms/step | 31.3 samples/s
|
| 788 |
+
Step 47110/50462 | Loss: 2.4844 | LR: 5.59e-06 | 1919.1ms/step | 31.3 samples/s
|
| 789 |
+
Step 47120/50462 | Loss: 2.4444 | LR: 5.56e-06 | 1920.5ms/step | 31.2 samples/s
|
| 790 |
+
Step 47130/50462 | Loss: 2.4195 | LR: 5.52e-06 | 1919.7ms/step | 31.3 samples/s
|
| 791 |
+
Step 47140/50462 | Loss: 2.4494 | LR: 5.49e-06 | 1919.1ms/step | 31.3 samples/s
|
| 792 |
+
Step 47150/50462 | Loss: 2.4933 | LR: 5.46e-06 | 1917.7ms/step | 31.3 samples/s
|
| 793 |
+
Step 47160/50462 | Loss: 2.4474 | LR: 5.42e-06 | 1919.0ms/step | 31.3 samples/s
|
| 794 |
+
Step 47170/50462 | Loss: 2.4742 | LR: 5.39e-06 | 1914.4ms/step | 31.3 samples/s
|
| 795 |
+
Step 47180/50462 | Loss: 2.4621 | LR: 5.36e-06 | 1917.5ms/step | 31.3 samples/s
|
| 796 |
+
Step 47190/50462 | Loss: 2.4557 | LR: 5.33e-06 | 1917.9ms/step | 31.3 samples/s
|
| 797 |
+
Step 47200/50462 | Loss: 2.4809 | LR: 5.29e-06 | 1916.7ms/step | 31.3 samples/s
|
| 798 |
+
Step 47210/50462 | Loss: 2.4132 | LR: 5.26e-06 | 1915.9ms/step | 31.3 samples/s
|
| 799 |
+
Step 47220/50462 | Loss: 2.4667 | LR: 5.23e-06 | 1918.4ms/step | 31.3 samples/s
|
| 800 |
+
Step 47230/50462 | Loss: 2.4535 | LR: 5.20e-06 | 1918.5ms/step | 31.3 samples/s
|
| 801 |
+
Step 47240/50462 | Loss: 2.4722 | LR: 5.16e-06 | 1921.0ms/step | 31.2 samples/s
|
| 802 |
+
Step 47250/50462 | Loss: 2.4480 | LR: 5.13e-06 | 1918.1ms/step | 31.3 samples/s
|
| 803 |
+
Validation at step 47250: Loss = 2.4988, Perplexity = 12.17
|
| 804 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47250
|
| 805 |
+
Step 47260/50462 | Loss: 2.4855 | LR: 5.10e-06 | 3246.2ms/step | 18.5 samples/s
|
| 806 |
+
Step 47270/50462 | Loss: 2.4627 | LR: 5.07e-06 | 1918.3ms/step | 31.3 samples/s
|
| 807 |
+
Step 47280/50462 | Loss: 2.4669 | LR: 5.04e-06 | 1919.0ms/step | 31.3 samples/s
|
| 808 |
+
Step 47290/50462 | Loss: 2.4736 | LR: 5.01e-06 | 1920.0ms/step | 31.3 samples/s
|
| 809 |
+
Step 47300/50462 | Loss: 2.4853 | LR: 4.97e-06 | 1915.0ms/step | 31.3 samples/s
|
| 810 |
+
Step 47310/50462 | Loss: 2.4795 | LR: 4.94e-06 | 1916.7ms/step | 31.3 samples/s
|
| 811 |
+
Step 47320/50462 | Loss: 2.4753 | LR: 4.91e-06 | 1917.1ms/step | 31.3 samples/s
|
| 812 |
+
Step 47330/50462 | Loss: 2.4399 | LR: 4.88e-06 | 1917.4ms/step | 31.3 samples/s
|
| 813 |
+
Step 47340/50462 | Loss: 2.4534 | LR: 4.85e-06 | 1916.5ms/step | 31.3 samples/s
|
| 814 |
+
Step 47350/50462 | Loss: 2.4654 | LR: 4.82e-06 | 1916.9ms/step | 31.3 samples/s
|
| 815 |
+
Step 47360/50462 | Loss: 2.4646 | LR: 4.79e-06 | 1916.5ms/step | 31.3 samples/s
|
| 816 |
+
Step 47370/50462 | Loss: 2.4799 | LR: 4.76e-06 | 1917.1ms/step | 31.3 samples/s
|
| 817 |
+
Step 47380/50462 | Loss: 2.5009 | LR: 4.73e-06 | 1919.5ms/step | 31.3 samples/s
|
| 818 |
+
Step 47390/50462 | Loss: 2.4429 | LR: 4.70e-06 | 1915.6ms/step | 31.3 samples/s
|
| 819 |
+
Step 47400/50462 | Loss: 2.4791 | LR: 4.67e-06 | 1915.1ms/step | 31.3 samples/s
|
| 820 |
+
Step 47410/50462 | Loss: 2.4885 | LR: 4.64e-06 | 1915.8ms/step | 31.3 samples/s
|
| 821 |
+
Step 47420/50462 | Loss: 2.4061 | LR: 4.61e-06 | 1917.6ms/step | 31.3 samples/s
|
| 822 |
+
Step 47430/50462 | Loss: 2.4439 | LR: 4.58e-06 | 1917.3ms/step | 31.3 samples/s
|
| 823 |
+
Step 47440/50462 | Loss: 2.4648 | LR: 4.55e-06 | 1919.3ms/step | 31.3 samples/s
|
| 824 |
+
Step 47450/50462 | Loss: 2.5031 | LR: 4.52e-06 | 1917.8ms/step | 31.3 samples/s
|
| 825 |
+
Step 47460/50462 | Loss: 2.4547 | LR: 4.49e-06 | 1919.5ms/step | 31.3 samples/s
|
| 826 |
+
Step 47470/50462 | Loss: 2.4539 | LR: 4.46e-06 | 1916.7ms/step | 31.3 samples/s
|
| 827 |
+
Step 47480/50462 | Loss: 2.4742 | LR: 4.43e-06 | 1916.8ms/step | 31.3 samples/s
|
| 828 |
+
Step 47490/50462 | Loss: 2.4467 | LR: 4.40e-06 | 1916.9ms/step | 31.3 samples/s
|
| 829 |
+
Step 47500/50462 | Loss: 2.4457 | LR: 4.37e-06 | 1919.1ms/step | 31.3 samples/s
|
| 830 |
+
Validation at step 47500: Loss = 2.4988, Perplexity = 12.17
|
| 831 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47500
|
| 832 |
+
Step 47510/50462 | Loss: 2.4867 | LR: 4.34e-06 | 3279.4ms/step | 18.3 samples/s
|
| 833 |
+
Step 47520/50462 | Loss: 2.4672 | LR: 4.31e-06 | 1917.2ms/step | 31.3 samples/s
|
| 834 |
+
Step 47530/50462 | Loss: 2.4825 | LR: 4.28e-06 | 1914.2ms/step | 31.3 samples/s
|
| 835 |
+
Step 47540/50462 | Loss: 2.5235 | LR: 4.25e-06 | 1917.8ms/step | 31.3 samples/s
|
| 836 |
+
Step 47550/50462 | Loss: 2.4669 | LR: 4.22e-06 | 1919.8ms/step | 31.3 samples/s
|
| 837 |
+
Step 47560/50462 | Loss: 2.4963 | LR: 4.19e-06 | 1919.6ms/step | 31.3 samples/s
|
| 838 |
+
Step 47570/50462 | Loss: 2.4652 | LR: 4.16e-06 | 1917.0ms/step | 31.3 samples/s
|
| 839 |
+
Step 47580/50462 | Loss: 2.4827 | LR: 4.13e-06 | 1914.1ms/step | 31.3 samples/s
|
| 840 |
+
Step 47590/50462 | Loss: 2.4916 | LR: 4.11e-06 | 1916.7ms/step | 31.3 samples/s
|
| 841 |
+
Step 47600/50462 | Loss: 2.4791 | LR: 4.08e-06 | 1916.5ms/step | 31.3 samples/s
|
| 842 |
+
Step 47610/50462 | Loss: 2.4443 | LR: 4.05e-06 | 1918.4ms/step | 31.3 samples/s
|
| 843 |
+
Step 47620/50462 | Loss: 2.4982 | LR: 4.02e-06 | 1918.0ms/step | 31.3 samples/s
|
| 844 |
+
Step 47630/50462 | Loss: 2.4646 | LR: 3.99e-06 | 1915.9ms/step | 31.3 samples/s
|
| 845 |
+
Step 47640/50462 | Loss: 2.4753 | LR: 3.97e-06 | 1916.3ms/step | 31.3 samples/s
|
| 846 |
+
Step 47650/50462 | Loss: 2.4668 | LR: 3.94e-06 | 1915.7ms/step | 31.3 samples/s
|
| 847 |
+
Step 47660/50462 | Loss: 2.3972 | LR: 3.91e-06 | 1918.7ms/step | 31.3 samples/s
|
| 848 |
+
Step 47670/50462 | Loss: 2.4377 | LR: 3.88e-06 | 1918.6ms/step | 31.3 samples/s
|
| 849 |
+
Step 47680/50462 | Loss: 2.4626 | LR: 3.85e-06 | 1916.1ms/step | 31.3 samples/s
|
| 850 |
+
Step 47690/50462 | Loss: 2.4618 | LR: 3.83e-06 | 1914.7ms/step | 31.3 samples/s
|
| 851 |
+
Step 47700/50462 | Loss: 2.4584 | LR: 3.80e-06 | 1914.0ms/step | 31.3 samples/s
|
| 852 |
+
Step 47710/50462 | Loss: 2.4769 | LR: 3.77e-06 | 1917.2ms/step | 31.3 samples/s
|
| 853 |
+
Step 47720/50462 | Loss: 2.4632 | LR: 3.74e-06 | 1915.7ms/step | 31.3 samples/s
|
| 854 |
+
Step 47730/50462 | Loss: 2.4365 | LR: 3.72e-06 | 1918.4ms/step | 31.3 samples/s
|
| 855 |
+
Step 47740/50462 | Loss: 2.5154 | LR: 3.69e-06 | 1918.2ms/step | 31.3 samples/s
|
| 856 |
+
Step 47750/50462 | Loss: 2.5042 | LR: 3.66e-06 | 1915.1ms/step | 31.3 samples/s
|
| 857 |
+
Validation at step 47750: Loss = 2.4988, Perplexity = 12.17
|
| 858 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-47750
|
| 859 |
+
Step 47760/50462 | Loss: 2.4718 | LR: 3.64e-06 | 3250.1ms/step | 18.5 samples/s
|
| 860 |
+
Step 47770/50462 | Loss: 2.4704 | LR: 3.61e-06 | 1919.6ms/step | 31.3 samples/s
|
| 861 |
+
Step 47780/50462 | Loss: 2.4749 | LR: 3.58e-06 | 1918.6ms/step | 31.3 samples/s
|
| 862 |
+
Step 47790/50462 | Loss: 2.4595 | LR: 3.56e-06 | 1921.7ms/step | 31.2 samples/s
|
| 863 |
+
Step 47800/50462 | Loss: 2.4364 | LR: 3.53e-06 | 1919.0ms/step | 31.3 samples/s
|
| 864 |
+
Step 47810/50462 | Loss: 2.4587 | LR: 3.50e-06 | 1916.3ms/step | 31.3 samples/s
|
| 865 |
+
Step 47820/50462 | Loss: 2.4801 | LR: 3.48e-06 | 1917.2ms/step | 31.3 samples/s
|
| 866 |
+
Step 47830/50462 | Loss: 2.4532 | LR: 3.45e-06 | 1921.1ms/step | 31.2 samples/s
|
| 867 |
+
Step 47840/50462 | Loss: 2.4763 | LR: 3.42e-06 | 1920.0ms/step | 31.2 samples/s
|
| 868 |
+
Step 47850/50462 | Loss: 2.4456 | LR: 3.40e-06 | 1918.8ms/step | 31.3 samples/s
|
| 869 |
+
Step 47860/50462 | Loss: 2.4704 | LR: 3.37e-06 | 1915.3ms/step | 31.3 samples/s
|
| 870 |
+
Step 47870/50462 | Loss: 2.5078 | LR: 3.35e-06 | 1917.8ms/step | 31.3 samples/s
|
| 871 |
+
Step 47880/50462 | Loss: 2.4665 | LR: 3.32e-06 | 1919.4ms/step | 31.3 samples/s
|
| 872 |
+
Step 47890/50462 | Loss: 2.4418 | LR: 3.30e-06 | 1917.9ms/step | 31.3 samples/s
|
| 873 |
+
Step 47900/50462 | Loss: 2.4607 | LR: 3.27e-06 | 1917.7ms/step | 31.3 samples/s
|
| 874 |
+
Step 47910/50462 | Loss: 2.5139 | LR: 3.24e-06 | 1919.3ms/step | 31.3 samples/s
|
| 875 |
+
Step 47920/50462 | Loss: 2.4614 | LR: 3.22e-06 | 1919.1ms/step | 31.3 samples/s
|
| 876 |
+
Step 47930/50462 | Loss: 2.4829 | LR: 3.19e-06 | 1917.7ms/step | 31.3 samples/s
|
| 877 |
+
Step 47940/50462 | Loss: 2.4583 | LR: 3.17e-06 | 1918.5ms/step | 31.3 samples/s
|
| 878 |
+
Step 47950/50462 | Loss: 2.4685 | LR: 3.14e-06 | 1916.2ms/step | 31.3 samples/s
|
| 879 |
+
Step 47960/50462 | Loss: 2.4592 | LR: 3.12e-06 | 1919.1ms/step | 31.3 samples/s
|
| 880 |
+
Step 47970/50462 | Loss: 2.4420 | LR: 3.09e-06 | 1919.2ms/step | 31.3 samples/s
|
| 881 |
+
Step 47980/50462 | Loss: 2.4619 | LR: 3.07e-06 | 1917.2ms/step | 31.3 samples/s
|
| 882 |
+
Step 47990/50462 | Loss: 2.4489 | LR: 3.04e-06 | 1918.7ms/step | 31.3 samples/s
|
| 883 |
+
Step 48000/50462 | Loss: 2.4927 | LR: 3.02e-06 | 1917.0ms/step | 31.3 samples/s
|
| 884 |
+
Validation at step 48000: Loss = 2.4988, Perplexity = 12.17
|
| 885 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48000
|
| 886 |
+
Step 48010/50462 | Loss: 2.4353 | LR: 3.00e-06 | 3264.6ms/step | 18.4 samples/s
|
| 887 |
+
Step 48020/50462 | Loss: 2.4998 | LR: 2.97e-06 | 1917.0ms/step | 31.3 samples/s
|
| 888 |
+
Step 48030/50462 | Loss: 2.4524 | LR: 2.95e-06 | 1920.8ms/step | 31.2 samples/s
|
| 889 |
+
Step 48040/50462 | Loss: 2.4857 | LR: 2.92e-06 | 1923.1ms/step | 31.2 samples/s
|
| 890 |
+
Step 48050/50462 | Loss: 2.4942 | LR: 2.90e-06 | 1923.7ms/step | 31.2 samples/s
|
| 891 |
+
Step 48060/50462 | Loss: 2.4287 | LR: 2.87e-06 | 1922.4ms/step | 31.2 samples/s
|
| 892 |
+
Step 48070/50462 | Loss: 2.4427 | LR: 2.85e-06 | 1923.2ms/step | 31.2 samples/s
|
| 893 |
+
Step 48080/50462 | Loss: 2.5093 | LR: 2.83e-06 | 1923.2ms/step | 31.2 samples/s
|
| 894 |
+
Step 48090/50462 | Loss: 2.4518 | LR: 2.80e-06 | 1922.8ms/step | 31.2 samples/s
|
| 895 |
+
Step 48100/50462 | Loss: 2.4384 | LR: 2.78e-06 | 1921.6ms/step | 31.2 samples/s
|
| 896 |
+
Step 48110/50462 | Loss: 2.4345 | LR: 2.76e-06 | 1923.8ms/step | 31.2 samples/s
|
| 897 |
+
Step 48120/50462 | Loss: 2.4388 | LR: 2.73e-06 | 1922.3ms/step | 31.2 samples/s
|
| 898 |
+
Step 48130/50462 | Loss: 2.5105 | LR: 2.71e-06 | 1922.3ms/step | 31.2 samples/s
|
| 899 |
+
Step 48140/50462 | Loss: 2.4855 | LR: 2.69e-06 | 1925.0ms/step | 31.2 samples/s
|
| 900 |
+
Step 48150/50462 | Loss: 2.4560 | LR: 2.66e-06 | 1923.3ms/step | 31.2 samples/s
|
| 901 |
+
Step 48160/50462 | Loss: 2.4398 | LR: 2.64e-06 | 1925.8ms/step | 31.2 samples/s
|
| 902 |
+
Step 48170/50462 | Loss: 2.4493 | LR: 2.62e-06 | 1926.3ms/step | 31.1 samples/s
|
| 903 |
+
Step 48180/50462 | Loss: 2.4542 | LR: 2.60e-06 | 1923.7ms/step | 31.2 samples/s
|
| 904 |
+
Step 48190/50462 | Loss: 2.4509 | LR: 2.57e-06 | 1924.1ms/step | 31.2 samples/s
|
| 905 |
+
Step 48200/50462 | Loss: 2.4344 | LR: 2.55e-06 | 1921.5ms/step | 31.2 samples/s
|
| 906 |
+
Step 48210/50462 | Loss: 2.4642 | LR: 2.53e-06 | 1925.2ms/step | 31.2 samples/s
|
| 907 |
+
Step 48220/50462 | Loss: 2.4660 | LR: 2.51e-06 | 1924.2ms/step | 31.2 samples/s
|
| 908 |
+
Step 48230/50462 | Loss: 2.4688 | LR: 2.48e-06 | 1925.1ms/step | 31.2 samples/s
|
| 909 |
+
Step 48240/50462 | Loss: 2.5138 | LR: 2.46e-06 | 1923.7ms/step | 31.2 samples/s
|
| 910 |
+
Step 48250/50462 | Loss: 2.4803 | LR: 2.44e-06 | 1923.0ms/step | 31.2 samples/s
|
| 911 |
+
Validation at step 48250: Loss = 2.4988, Perplexity = 12.17
|
| 912 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48250
|
| 913 |
+
Step 48260/50462 | Loss: 2.4407 | LR: 2.42e-06 | 3253.6ms/step | 18.4 samples/s
|
| 914 |
+
Step 48270/50462 | Loss: 2.4776 | LR: 2.39e-06 | 1913.4ms/step | 31.4 samples/s
|
| 915 |
+
Step 48280/50462 | Loss: 2.4794 | LR: 2.37e-06 | 1918.6ms/step | 31.3 samples/s
|
| 916 |
+
Step 48290/50462 | Loss: 2.4778 | LR: 2.35e-06 | 1917.6ms/step | 31.3 samples/s
|
| 917 |
+
Step 48300/50462 | Loss: 2.4407 | LR: 2.33e-06 | 1914.1ms/step | 31.3 samples/s
|
| 918 |
+
Step 48310/50462 | Loss: 2.4481 | LR: 2.31e-06 | 1918.3ms/step | 31.3 samples/s
|
| 919 |
+
Step 48320/50462 | Loss: 2.4377 | LR: 2.29e-06 | 1918.9ms/step | 31.3 samples/s
|
| 920 |
+
Step 48330/50462 | Loss: 2.5010 | LR: 2.27e-06 | 1920.3ms/step | 31.2 samples/s
|
| 921 |
+
Step 48340/50462 | Loss: 2.4903 | LR: 2.24e-06 | 1916.2ms/step | 31.3 samples/s
|
| 922 |
+
Step 48350/50462 | Loss: 2.4931 | LR: 2.22e-06 | 1917.0ms/step | 31.3 samples/s
|
| 923 |
+
Step 48360/50462 | Loss: 2.4656 | LR: 2.20e-06 | 1917.5ms/step | 31.3 samples/s
|
| 924 |
+
Step 48370/50462 | Loss: 2.4389 | LR: 2.18e-06 | 1919.1ms/step | 31.3 samples/s
|
| 925 |
+
Step 48380/50462 | Loss: 2.5167 | LR: 2.16e-06 | 1920.9ms/step | 31.2 samples/s
|
| 926 |
+
Step 48390/50462 | Loss: 2.4923 | LR: 2.14e-06 | 1918.0ms/step | 31.3 samples/s
|
| 927 |
+
Step 48400/50462 | Loss: 2.4492 | LR: 2.12e-06 | 1917.7ms/step | 31.3 samples/s
|
| 928 |
+
Step 48410/50462 | Loss: 2.4649 | LR: 2.10e-06 | 1919.0ms/step | 31.3 samples/s
|
| 929 |
+
Step 48420/50462 | Loss: 2.4215 | LR: 2.08e-06 | 1918.6ms/step | 31.3 samples/s
|
| 930 |
+
Step 48430/50462 | Loss: 2.4341 | LR: 2.06e-06 | 1919.2ms/step | 31.3 samples/s
|
| 931 |
+
Step 48440/50462 | Loss: 2.3954 | LR: 2.04e-06 | 1920.9ms/step | 31.2 samples/s
|
| 932 |
+
Step 48450/50462 | Loss: 2.4823 | LR: 2.02e-06 | 1917.7ms/step | 31.3 samples/s
|
| 933 |
+
Step 48460/50462 | Loss: 2.4719 | LR: 2.00e-06 | 1919.9ms/step | 31.3 samples/s
|
| 934 |
+
Step 48470/50462 | Loss: 2.4613 | LR: 1.98e-06 | 1919.7ms/step | 31.3 samples/s
|
| 935 |
+
Step 48480/50462 | Loss: 2.4654 | LR: 1.96e-06 | 1918.4ms/step | 31.3 samples/s
|
| 936 |
+
Step 48490/50462 | Loss: 2.5074 | LR: 1.94e-06 | 1920.5ms/step | 31.2 samples/s
|
| 937 |
+
Step 48500/50462 | Loss: 2.4540 | LR: 1.92e-06 | 1920.0ms/step | 31.2 samples/s
|
| 938 |
+
Validation at step 48500: Loss = 2.4989, Perplexity = 12.17
|
| 939 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48500
|
| 940 |
+
Step 48510/50462 | Loss: 2.4430 | LR: 1.90e-06 | 3248.8ms/step | 18.5 samples/s
|
| 941 |
+
Step 48520/50462 | Loss: 2.4406 | LR: 1.88e-06 | 1915.7ms/step | 31.3 samples/s
|
| 942 |
+
Step 48530/50462 | Loss: 2.4425 | LR: 1.86e-06 | 1919.8ms/step | 31.3 samples/s
|
| 943 |
+
Step 48540/50462 | Loss: 2.4552 | LR: 1.84e-06 | 1915.2ms/step | 31.3 samples/s
|
| 944 |
+
Step 48550/50462 | Loss: 2.4557 | LR: 1.82e-06 | 1918.6ms/step | 31.3 samples/s
|
| 945 |
+
Step 48560/50462 | Loss: 2.4222 | LR: 1.80e-06 | 1913.8ms/step | 31.4 samples/s
|
| 946 |
+
Step 48570/50462 | Loss: 2.4386 | LR: 1.78e-06 | 1917.3ms/step | 31.3 samples/s
|
| 947 |
+
Step 48580/50462 | Loss: 2.4402 | LR: 1.77e-06 | 1917.4ms/step | 31.3 samples/s
|
| 948 |
+
Step 48590/50462 | Loss: 2.4742 | LR: 1.75e-06 | 1918.0ms/step | 31.3 samples/s
|
| 949 |
+
Step 48600/50462 | Loss: 2.4750 | LR: 1.73e-06 | 1917.5ms/step | 31.3 samples/s
|
| 950 |
+
Step 48610/50462 | Loss: 2.5085 | LR: 1.71e-06 | 1916.2ms/step | 31.3 samples/s
|
| 951 |
+
Step 48620/50462 | Loss: 2.4287 | LR: 1.69e-06 | 1916.2ms/step | 31.3 samples/s
|
| 952 |
+
Step 48630/50462 | Loss: 2.5159 | LR: 1.67e-06 | 1918.0ms/step | 31.3 samples/s
|
| 953 |
+
Step 48640/50462 | Loss: 2.4347 | LR: 1.66e-06 | 1917.7ms/step | 31.3 samples/s
|
| 954 |
+
Step 48650/50462 | Loss: 2.4472 | LR: 1.64e-06 | 1917.9ms/step | 31.3 samples/s
|
| 955 |
+
Step 48660/50462 | Loss: 2.4389 | LR: 1.62e-06 | 1919.0ms/step | 31.3 samples/s
|
| 956 |
+
Step 48670/50462 | Loss: 2.4259 | LR: 1.60e-06 | 1919.3ms/step | 31.3 samples/s
|
| 957 |
+
Step 48680/50462 | Loss: 2.4301 | LR: 1.58e-06 | 1917.9ms/step | 31.3 samples/s
|
| 958 |
+
Step 48690/50462 | Loss: 2.4910 | LR: 1.57e-06 | 1916.7ms/step | 31.3 samples/s
|
| 959 |
+
Step 48700/50462 | Loss: 2.4669 | LR: 1.55e-06 | 1916.6ms/step | 31.3 samples/s
|
| 960 |
+
Step 48710/50462 | Loss: 2.4263 | LR: 1.53e-06 | 1918.7ms/step | 31.3 samples/s
|
| 961 |
+
Step 48720/50462 | Loss: 2.4289 | LR: 1.51e-06 | 1920.3ms/step | 31.2 samples/s
|
| 962 |
+
Step 48730/50462 | Loss: 2.4532 | LR: 1.50e-06 | 1918.8ms/step | 31.3 samples/s
|
| 963 |
+
Step 48740/50462 | Loss: 2.4840 | LR: 1.48e-06 | 1917.3ms/step | 31.3 samples/s
|
| 964 |
+
Step 48750/50462 | Loss: 2.4612 | LR: 1.46e-06 | 1917.5ms/step | 31.3 samples/s
|
| 965 |
+
Validation at step 48750: Loss = 2.4989, Perplexity = 12.17
|
| 966 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-48750
|
| 967 |
+
Step 48760/50462 | Loss: 2.4310 | LR: 1.44e-06 | 3245.9ms/step | 18.5 samples/s
|
| 968 |
+
Step 48770/50462 | Loss: 2.4619 | LR: 1.43e-06 | 1913.7ms/step | 31.4 samples/s
|
| 969 |
+
Step 48780/50462 | Loss: 2.4302 | LR: 1.41e-06 | 1918.1ms/step | 31.3 samples/s
|
| 970 |
+
Step 48790/50462 | Loss: 2.4586 | LR: 1.39e-06 | 1920.8ms/step | 31.2 samples/s
|
| 971 |
+
Step 48800/50462 | Loss: 2.4736 | LR: 1.38e-06 | 1922.9ms/step | 31.2 samples/s
|
| 972 |
+
Step 48810/50462 | Loss: 2.4688 | LR: 1.36e-06 | 1923.2ms/step | 31.2 samples/s
|
| 973 |
+
Step 48820/50462 | Loss: 2.4595 | LR: 1.34e-06 | 1922.3ms/step | 31.2 samples/s
|
| 974 |
+
Step 48830/50462 | Loss: 2.4388 | LR: 1.33e-06 | 1924.2ms/step | 31.2 samples/s
|
| 975 |
+
Step 48840/50462 | Loss: 2.5066 | LR: 1.31e-06 | 1922.7ms/step | 31.2 samples/s
|
| 976 |
+
Step 48850/50462 | Loss: 2.4724 | LR: 1.30e-06 | 1921.5ms/step | 31.2 samples/s
|
| 977 |
+
Step 48860/50462 | Loss: 2.4585 | LR: 1.28e-06 | 1917.0ms/step | 31.3 samples/s
|
| 978 |
+
Step 48870/50462 | Loss: 2.5023 | LR: 1.26e-06 | 1918.4ms/step | 31.3 samples/s
|
| 979 |
+
Step 48880/50462 | Loss: 2.4467 | LR: 1.25e-06 | 1918.8ms/step | 31.3 samples/s
|
| 980 |
+
Step 48890/50462 | Loss: 2.4677 | LR: 1.23e-06 | 1916.5ms/step | 31.3 samples/s
|
| 981 |
+
Step 48900/50462 | Loss: 2.4269 | LR: 1.22e-06 | 1919.1ms/step | 31.3 samples/s
|
| 982 |
+
Step 48910/50462 | Loss: 2.4301 | LR: 1.20e-06 | 1919.7ms/step | 31.3 samples/s
|
| 983 |
+
Step 48920/50462 | Loss: 2.4694 | LR: 1.19e-06 | 1917.5ms/step | 31.3 samples/s
|
| 984 |
+
Step 48930/50462 | Loss: 2.4361 | LR: 1.17e-06 | 1920.1ms/step | 31.2 samples/s
|
| 985 |
+
Step 48940/50462 | Loss: 2.4731 | LR: 1.16e-06 | 1918.7ms/step | 31.3 samples/s
|
| 986 |
+
Step 48950/50462 | Loss: 2.4650 | LR: 1.14e-06 | 1915.6ms/step | 31.3 samples/s
|
| 987 |
+
Step 48960/50462 | Loss: 2.4940 | LR: 1.13e-06 | 1917.8ms/step | 31.3 samples/s
|
| 988 |
+
Step 48970/50462 | Loss: 2.4244 | LR: 1.11e-06 | 1918.6ms/step | 31.3 samples/s
|
| 989 |
+
Step 48980/50462 | Loss: 2.4672 | LR: 1.10e-06 | 1916.5ms/step | 31.3 samples/s
|
| 990 |
+
Step 48990/50462 | Loss: 2.4576 | LR: 1.08e-06 | 1920.0ms/step | 31.2 samples/s
|
| 991 |
+
Step 49000/50462 | Loss: 2.4554 | LR: 1.07e-06 | 1919.7ms/step | 31.3 samples/s
|
| 992 |
+
Validation at step 49000: Loss = 2.4989, Perplexity = 12.17
|
| 993 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49000
|
| 994 |
+
Step 49010/50462 | Loss: 2.4918 | LR: 1.05e-06 | 3249.7ms/step | 18.5 samples/s
|
| 995 |
+
Step 49020/50462 | Loss: 2.4728 | LR: 1.04e-06 | 1915.5ms/step | 31.3 samples/s
|
| 996 |
+
Step 49030/50462 | Loss: 2.4415 | LR: 1.02e-06 | 1920.6ms/step | 31.2 samples/s
|
| 997 |
+
Step 49040/50462 | Loss: 2.4416 | LR: 1.01e-06 | 1923.0ms/step | 31.2 samples/s
|
| 998 |
+
Step 49050/50462 | Loss: 2.4504 | LR: 9.95e-07 | 1921.9ms/step | 31.2 samples/s
|
| 999 |
+
Step 49060/50462 | Loss: 2.4626 | LR: 9.81e-07 | 1917.4ms/step | 31.3 samples/s
|
| 1000 |
+
Step 49070/50462 | Loss: 2.5310 | LR: 9.67e-07 | 1918.8ms/step | 31.3 samples/s
|
| 1001 |
+
Step 49080/50462 | Loss: 2.4718 | LR: 9.53e-07 | 1920.6ms/step | 31.2 samples/s
|
| 1002 |
+
Step 49090/50462 | Loss: 2.4717 | LR: 9.39e-07 | 1916.2ms/step | 31.3 samples/s
|
| 1003 |
+
Step 49100/50462 | Loss: 2.4692 | LR: 9.25e-07 | 1915.1ms/step | 31.3 samples/s
|
| 1004 |
+
Step 49110/50462 | Loss: 2.4824 | LR: 9.12e-07 | 1916.9ms/step | 31.3 samples/s
|
| 1005 |
+
Step 49120/50462 | Loss: 2.4652 | LR: 8.99e-07 | 1917.9ms/step | 31.3 samples/s
|
| 1006 |
+
Step 49130/50462 | Loss: 2.4763 | LR: 8.85e-07 | 1917.7ms/step | 31.3 samples/s
|
| 1007 |
+
Step 49140/50462 | Loss: 2.4745 | LR: 8.72e-07 | 1915.5ms/step | 31.3 samples/s
|
| 1008 |
+
Step 49150/50462 | Loss: 2.4613 | LR: 8.59e-07 | 1919.5ms/step | 31.3 samples/s
|
| 1009 |
+
Step 49160/50462 | Loss: 2.4523 | LR: 8.46e-07 | 1920.1ms/step | 31.2 samples/s
|
| 1010 |
+
Step 49170/50462 | Loss: 2.4351 | LR: 8.33e-07 | 1917.8ms/step | 31.3 samples/s
|
| 1011 |
+
Step 49180/50462 | Loss: 2.4550 | LR: 8.20e-07 | 1914.5ms/step | 31.3 samples/s
|
| 1012 |
+
Step 49190/50462 | Loss: 2.4864 | LR: 8.07e-07 | 1918.3ms/step | 31.3 samples/s
|
| 1013 |
+
Step 49200/50462 | Loss: 2.4399 | LR: 7.95e-07 | 1916.4ms/step | 31.3 samples/s
|
| 1014 |
+
Step 49210/50462 | Loss: 2.4556 | LR: 7.82e-07 | 1916.4ms/step | 31.3 samples/s
|
| 1015 |
+
Step 49220/50462 | Loss: 2.4866 | LR: 7.70e-07 | 1916.8ms/step | 31.3 samples/s
|
| 1016 |
+
Step 49230/50462 | Loss: 2.4462 | LR: 7.57e-07 | 1915.8ms/step | 31.3 samples/s
|
| 1017 |
+
Step 49240/50462 | Loss: 2.4963 | LR: 7.45e-07 | 1917.7ms/step | 31.3 samples/s
|
| 1018 |
+
Step 49250/50462 | Loss: 2.4353 | LR: 7.33e-07 | 1918.4ms/step | 31.3 samples/s
|
| 1019 |
+
Validation at step 49250: Loss = 2.4989, Perplexity = 12.17
|
| 1020 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49250
|
| 1021 |
+
Step 49260/50462 | Loss: 2.4547 | LR: 7.21e-07 | 3329.2ms/step | 18.0 samples/s
|
| 1022 |
+
Step 49270/50462 | Loss: 2.4553 | LR: 7.09e-07 | 1919.7ms/step | 31.3 samples/s
|
| 1023 |
+
Step 49280/50462 | Loss: 2.4529 | LR: 6.97e-07 | 1919.3ms/step | 31.3 samples/s
|
| 1024 |
+
Step 49290/50462 | Loss: 2.4322 | LR: 6.85e-07 | 1923.9ms/step | 31.2 samples/s
|
| 1025 |
+
Step 49300/50462 | Loss: 2.4783 | LR: 6.74e-07 | 1920.4ms/step | 31.2 samples/s
|
| 1026 |
+
Step 49310/50462 | Loss: 2.4220 | LR: 6.62e-07 | 1921.1ms/step | 31.2 samples/s
|
| 1027 |
+
Step 49320/50462 | Loss: 2.4773 | LR: 6.51e-07 | 1917.9ms/step | 31.3 samples/s
|
| 1028 |
+
Step 49330/50462 | Loss: 2.4386 | LR: 6.39e-07 | 1915.7ms/step | 31.3 samples/s
|
| 1029 |
+
Step 49340/50462 | Loss: 2.4521 | LR: 6.28e-07 | 1916.7ms/step | 31.3 samples/s
|
| 1030 |
+
Step 49350/50462 | Loss: 2.4953 | LR: 6.17e-07 | 1919.7ms/step | 31.3 samples/s
|
| 1031 |
+
Step 49360/50462 | Loss: 2.5013 | LR: 6.06e-07 | 1919.9ms/step | 31.3 samples/s
|
| 1032 |
+
Step 49370/50462 | Loss: 2.4876 | LR: 5.95e-07 | 1918.2ms/step | 31.3 samples/s
|
| 1033 |
+
Step 49380/50462 | Loss: 2.4638 | LR: 5.84e-07 | 1917.4ms/step | 31.3 samples/s
|
| 1034 |
+
Step 49390/50462 | Loss: 2.4245 | LR: 5.73e-07 | 1922.5ms/step | 31.2 samples/s
|
| 1035 |
+
Step 49400/50462 | Loss: 2.4566 | LR: 5.63e-07 | 1921.0ms/step | 31.2 samples/s
|
| 1036 |
+
Step 49410/50462 | Loss: 2.4551 | LR: 5.52e-07 | 1919.1ms/step | 31.3 samples/s
|
| 1037 |
+
Step 49420/50462 | Loss: 2.4716 | LR: 5.42e-07 | 1917.6ms/step | 31.3 samples/s
|
| 1038 |
+
Step 49430/50462 | Loss: 2.4680 | LR: 5.31e-07 | 1918.9ms/step | 31.3 samples/s
|
| 1039 |
+
Step 49440/50462 | Loss: 2.4413 | LR: 5.21e-07 | 1917.3ms/step | 31.3 samples/s
|
| 1040 |
+
Step 49450/50462 | Loss: 2.4589 | LR: 5.11e-07 | 1916.9ms/step | 31.3 samples/s
|
| 1041 |
+
Step 49460/50462 | Loss: 2.4852 | LR: 5.01e-07 | 1918.9ms/step | 31.3 samples/s
|
| 1042 |
+
Step 49470/50462 | Loss: 2.4431 | LR: 4.91e-07 | 1920.2ms/step | 31.2 samples/s
|
| 1043 |
+
Step 49480/50462 | Loss: 2.4691 | LR: 4.81e-07 | 1917.8ms/step | 31.3 samples/s
|
| 1044 |
+
Step 49490/50462 | Loss: 2.4631 | LR: 4.72e-07 | 1919.6ms/step | 31.3 samples/s
|
| 1045 |
+
Step 49500/50462 | Loss: 2.4153 | LR: 4.62e-07 | 1918.6ms/step | 31.3 samples/s
|
| 1046 |
+
Validation at step 49500: Loss = 2.4989, Perplexity = 12.17
|
| 1047 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49500
|
| 1048 |
+
Step 49510/50462 | Loss: 2.4739 | LR: 4.52e-07 | 3253.7ms/step | 18.4 samples/s
|
| 1049 |
+
Step 49520/50462 | Loss: 2.4585 | LR: 4.43e-07 | 1916.6ms/step | 31.3 samples/s
|
| 1050 |
+
Step 49530/50462 | Loss: 2.4405 | LR: 4.34e-07 | 1920.0ms/step | 31.3 samples/s
|
| 1051 |
+
Step 49540/50462 | Loss: 2.4827 | LR: 4.24e-07 | 1920.8ms/step | 31.2 samples/s
|
| 1052 |
+
Step 49550/50462 | Loss: 2.4285 | LR: 4.15e-07 | 1921.7ms/step | 31.2 samples/s
|
| 1053 |
+
Step 49560/50462 | Loss: 2.4084 | LR: 4.06e-07 | 1922.7ms/step | 31.2 samples/s
|
| 1054 |
+
Step 49570/50462 | Loss: 2.4576 | LR: 3.97e-07 | 1922.2ms/step | 31.2 samples/s
|
| 1055 |
+
Step 49580/50462 | Loss: 2.4087 | LR: 3.88e-07 | 1922.6ms/step | 31.2 samples/s
|
| 1056 |
+
Step 49590/50462 | Loss: 2.4472 | LR: 3.79e-07 | 1923.9ms/step | 31.2 samples/s
|
| 1057 |
+
Step 49600/50462 | Loss: 2.4458 | LR: 3.71e-07 | 1924.2ms/step | 31.2 samples/s
|
| 1058 |
+
Step 49610/50462 | Loss: 2.4779 | LR: 3.62e-07 | 1924.7ms/step | 31.2 samples/s
|
| 1059 |
+
Step 49620/50462 | Loss: 2.4345 | LR: 3.54e-07 | 1923.4ms/step | 31.2 samples/s
|
| 1060 |
+
Step 49630/50462 | Loss: 2.4247 | LR: 3.45e-07 | 1922.7ms/step | 31.2 samples/s
|
| 1061 |
+
Step 49640/50462 | Loss: 2.4311 | LR: 3.37e-07 | 1923.9ms/step | 31.2 samples/s
|
| 1062 |
+
Step 49650/50462 | Loss: 2.4190 | LR: 3.29e-07 | 1922.6ms/step | 31.2 samples/s
|
| 1063 |
+
Step 49660/50462 | Loss: 2.4283 | LR: 3.21e-07 | 1920.0ms/step | 31.2 samples/s
|
| 1064 |
+
Step 49670/50462 | Loss: 2.4728 | LR: 3.13e-07 | 1924.0ms/step | 31.2 samples/s
|
| 1065 |
+
Step 49680/50462 | Loss: 2.4499 | LR: 3.05e-07 | 1925.0ms/step | 31.2 samples/s
|
| 1066 |
+
Step 49690/50462 | Loss: 2.4583 | LR: 2.97e-07 | 1923.0ms/step | 31.2 samples/s
|
| 1067 |
+
Step 49700/50462 | Loss: 2.4532 | LR: 2.90e-07 | 1924.4ms/step | 31.2 samples/s
|
| 1068 |
+
Step 49710/50462 | Loss: 2.4434 | LR: 2.82e-07 | 1919.9ms/step | 31.3 samples/s
|
| 1069 |
+
Step 49720/50462 | Loss: 2.4492 | LR: 2.75e-07 | 1921.1ms/step | 31.2 samples/s
|
| 1070 |
+
Step 49730/50462 | Loss: 2.4780 | LR: 2.67e-07 | 1922.4ms/step | 31.2 samples/s
|
| 1071 |
+
Step 49740/50462 | Loss: 2.4489 | LR: 2.60e-07 | 1923.5ms/step | 31.2 samples/s
|
| 1072 |
+
Step 49750/50462 | Loss: 2.4648 | LR: 2.53e-07 | 1924.2ms/step | 31.2 samples/s
|
| 1073 |
+
Validation at step 49750: Loss = 2.4989, Perplexity = 12.17
|
| 1074 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-49750
|
| 1075 |
+
Step 49760/50462 | Loss: 2.4704 | LR: 2.46e-07 | 3263.1ms/step | 18.4 samples/s
|
| 1076 |
+
Step 49770/50462 | Loss: 2.4019 | LR: 2.39e-07 | 1915.4ms/step | 31.3 samples/s
|
| 1077 |
+
Step 49780/50462 | Loss: 2.4313 | LR: 2.32e-07 | 1919.8ms/step | 31.3 samples/s
|
| 1078 |
+
Step 49790/50462 | Loss: 2.5284 | LR: 2.25e-07 | 1917.8ms/step | 31.3 samples/s
|
| 1079 |
+
Step 49800/50462 | Loss: 2.4334 | LR: 2.19e-07 | 1916.2ms/step | 31.3 samples/s
|
| 1080 |
+
Step 49810/50462 | Loss: 2.4540 | LR: 2.12e-07 | 1917.0ms/step | 31.3 samples/s
|
| 1081 |
+
Step 49820/50462 | Loss: 2.4802 | LR: 2.06e-07 | 1916.8ms/step | 31.3 samples/s
|
| 1082 |
+
Step 49830/50462 | Loss: 2.4738 | LR: 1.99e-07 | 1916.0ms/step | 31.3 samples/s
|
| 1083 |
+
Step 49840/50462 | Loss: 2.4790 | LR: 1.93e-07 | 1917.8ms/step | 31.3 samples/s
|
| 1084 |
+
Step 49850/50462 | Loss: 2.4824 | LR: 1.87e-07 | 1918.4ms/step | 31.3 samples/s
|
| 1085 |
+
Step 49860/50462 | Loss: 2.4529 | LR: 1.81e-07 | 1921.7ms/step | 31.2 samples/s
|
| 1086 |
+
Step 49870/50462 | Loss: 2.5313 | LR: 1.75e-07 | 1917.6ms/step | 31.3 samples/s
|
| 1087 |
+
Step 49880/50462 | Loss: 2.4425 | LR: 1.69e-07 | 1920.2ms/step | 31.2 samples/s
|
| 1088 |
+
Step 49890/50462 | Loss: 2.4782 | LR: 1.63e-07 | 1919.7ms/step | 31.3 samples/s
|
| 1089 |
+
Step 49900/50462 | Loss: 2.4688 | LR: 1.58e-07 | 1919.0ms/step | 31.3 samples/s
|
| 1090 |
+
Step 49910/50462 | Loss: 2.4431 | LR: 1.52e-07 | 1919.4ms/step | 31.3 samples/s
|
| 1091 |
+
Step 49920/50462 | Loss: 2.4735 | LR: 1.47e-07 | 1919.3ms/step | 31.3 samples/s
|
| 1092 |
+
Step 49930/50462 | Loss: 2.4415 | LR: 1.41e-07 | 1917.8ms/step | 31.3 samples/s
|
| 1093 |
+
Step 49940/50462 | Loss: 2.4553 | LR: 1.36e-07 | 1918.8ms/step | 31.3 samples/s
|
| 1094 |
+
Step 49950/50462 | Loss: 2.4586 | LR: 1.31e-07 | 1917.3ms/step | 31.3 samples/s
|
| 1095 |
+
Step 49960/50462 | Loss: 2.4505 | LR: 1.26e-07 | 1920.3ms/step | 31.2 samples/s
|
| 1096 |
+
Step 49970/50462 | Loss: 2.4707 | LR: 1.21e-07 | 1917.7ms/step | 31.3 samples/s
|
| 1097 |
+
Step 49980/50462 | Loss: 2.4379 | LR: 1.16e-07 | 1919.7ms/step | 31.3 samples/s
|
| 1098 |
+
Step 49990/50462 | Loss: 2.5049 | LR: 1.11e-07 | 1920.3ms/step | 31.2 samples/s
|
| 1099 |
+
Step 50000/50462 | Loss: 2.4293 | LR: 1.07e-07 | 1920.7ms/step | 31.2 samples/s
|
| 1100 |
+
Validation at step 50000: Loss = 2.4989, Perplexity = 12.17
|
| 1101 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50000
|
| 1102 |
+
Step 50010/50462 | Loss: 2.4104 | LR: 1.02e-07 | 3251.5ms/step | 18.5 samples/s
|
| 1103 |
+
Step 50020/50462 | Loss: 2.4556 | LR: 9.75e-08 | 1920.0ms/step | 31.2 samples/s
|
| 1104 |
+
Step 50030/50462 | Loss: 2.4608 | LR: 9.32e-08 | 1920.2ms/step | 31.2 samples/s
|
| 1105 |
+
Step 50040/50462 | Loss: 2.4836 | LR: 8.89e-08 | 1920.4ms/step | 31.2 samples/s
|
| 1106 |
+
Step 50050/50462 | Loss: 2.5032 | LR: 8.47e-08 | 1923.8ms/step | 31.2 samples/s
|
| 1107 |
+
Step 50060/50462 | Loss: 2.4885 | LR: 8.07e-08 | 1924.3ms/step | 31.2 samples/s
|
| 1108 |
+
Step 50070/50462 | Loss: 2.4883 | LR: 7.67e-08 | 1922.0ms/step | 31.2 samples/s
|
| 1109 |
+
Step 50080/50462 | Loss: 2.4687 | LR: 7.28e-08 | 1922.3ms/step | 31.2 samples/s
|
| 1110 |
+
Step 50090/50462 | Loss: 2.4525 | LR: 6.91e-08 | 1923.5ms/step | 31.2 samples/s
|
| 1111 |
+
Step 50100/50462 | Loss: 2.4403 | LR: 6.54e-08 | 1925.6ms/step | 31.2 samples/s
|
| 1112 |
+
Step 50110/50462 | Loss: 2.4738 | LR: 6.19e-08 | 1924.8ms/step | 31.2 samples/s
|
| 1113 |
+
Step 50120/50462 | Loss: 2.4606 | LR: 5.84e-08 | 1921.1ms/step | 31.2 samples/s
|
| 1114 |
+
Step 50130/50462 | Loss: 2.4388 | LR: 5.50e-08 | 1923.9ms/step | 31.2 samples/s
|
| 1115 |
+
Step 50140/50462 | Loss: 2.4341 | LR: 5.18e-08 | 1920.7ms/step | 31.2 samples/s
|
| 1116 |
+
Step 50150/50462 | Loss: 2.4438 | LR: 4.86e-08 | 1922.6ms/step | 31.2 samples/s
|
| 1117 |
+
Step 50160/50462 | Loss: 2.4943 | LR: 4.55e-08 | 1923.6ms/step | 31.2 samples/s
|
| 1118 |
+
Step 50170/50462 | Loss: 2.4167 | LR: 4.26e-08 | 1924.5ms/step | 31.2 samples/s
|
| 1119 |
+
Step 50180/50462 | Loss: 2.4217 | LR: 3.97e-08 | 1922.9ms/step | 31.2 samples/s
|
| 1120 |
+
Step 50190/50462 | Loss: 2.4289 | LR: 3.69e-08 | 1922.2ms/step | 31.2 samples/s
|
| 1121 |
+
Step 50200/50462 | Loss: 2.4397 | LR: 3.43e-08 | 1921.6ms/step | 31.2 samples/s
|
| 1122 |
+
Step 50210/50462 | Loss: 2.4460 | LR: 3.17e-08 | 1919.7ms/step | 31.3 samples/s
|
| 1123 |
+
Step 50220/50462 | Loss: 2.4347 | LR: 2.92e-08 | 1922.1ms/step | 31.2 samples/s
|
| 1124 |
+
Step 50230/50462 | Loss: 2.4324 | LR: 2.69e-08 | 1920.3ms/step | 31.2 samples/s
|
| 1125 |
+
Step 50240/50462 | Loss: 2.4404 | LR: 2.46e-08 | 1922.8ms/step | 31.2 samples/s
|
| 1126 |
+
Step 50250/50462 | Loss: 2.4152 | LR: 2.24e-08 | 1923.0ms/step | 31.2 samples/s
|
| 1127 |
+
Validation at step 50250: Loss = 2.4989, Perplexity = 12.17
|
| 1128 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50250
|
| 1129 |
+
Step 50260/50462 | Loss: 2.4820 | LR: 2.04e-08 | 3297.3ms/step | 18.2 samples/s
|
| 1130 |
+
Step 50270/50462 | Loss: 2.4292 | LR: 1.84e-08 | 1914.0ms/step | 31.3 samples/s
|
| 1131 |
+
Step 50280/50462 | Loss: 2.4780 | LR: 1.65e-08 | 1916.3ms/step | 31.3 samples/s
|
| 1132 |
+
Step 50290/50462 | Loss: 2.4465 | LR: 1.48e-08 | 1920.7ms/step | 31.2 samples/s
|
| 1133 |
+
Step 50300/50462 | Loss: 2.4276 | LR: 1.31e-08 | 1918.2ms/step | 31.3 samples/s
|
| 1134 |
+
Step 50310/50462 | Loss: 2.4382 | LR: 1.15e-08 | 1916.8ms/step | 31.3 samples/s
|
| 1135 |
+
Step 50320/50462 | Loss: 2.4418 | LR: 1.01e-08 | 1914.9ms/step | 31.3 samples/s
|
| 1136 |
+
Step 50330/50462 | Loss: 2.4617 | LR: 8.70e-09 | 1918.3ms/step | 31.3 samples/s
|
| 1137 |
+
Step 50340/50462 | Loss: 2.4766 | LR: 7.43e-09 | 1915.3ms/step | 31.3 samples/s
|
| 1138 |
+
Step 50350/50462 | Loss: 2.4444 | LR: 6.26e-09 | 1917.6ms/step | 31.3 samples/s
|
| 1139 |
+
Step 50360/50462 | Loss: 2.4418 | LR: 5.19e-09 | 1919.7ms/step | 31.3 samples/s
|
| 1140 |
+
Step 50370/50462 | Loss: 2.5199 | LR: 4.23e-09 | 1917.4ms/step | 31.3 samples/s
|
| 1141 |
+
Step 50380/50462 | Loss: 2.4488 | LR: 3.36e-09 | 1920.1ms/step | 31.2 samples/s
|
| 1142 |
+
Step 50390/50462 | Loss: 2.4309 | LR: 2.59e-09 | 1921.6ms/step | 31.2 samples/s
|
| 1143 |
+
Step 50400/50462 | Loss: 2.4418 | LR: 1.92e-09 | 1916.5ms/step | 31.3 samples/s
|
| 1144 |
+
Step 50410/50462 | Loss: 2.4476 | LR: 1.35e-09 | 1919.0ms/step | 31.3 samples/s
|
| 1145 |
+
Step 50420/50462 | Loss: 2.4333 | LR: 8.81e-10 | 1916.2ms/step | 31.3 samples/s
|
| 1146 |
+
Step 50430/50462 | Loss: 2.4399 | LR: 5.11e-10 | 1916.0ms/step | 31.3 samples/s
|
| 1147 |
+
Step 50440/50462 | Loss: 2.4397 | LR: 2.42e-10 | 1920.8ms/step | 31.2 samples/s
|
| 1148 |
+
Step 50450/50462 | Loss: 2.4503 | LR: 7.19e-11 | 1920.5ms/step | 31.2 samples/s
|
| 1149 |
+
Step 50460/50462 | Loss: 2.4393 | LR: 2.00e-12 | 1917.9ms/step | 31.3 samples/s
|
| 1150 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-50462
|
| 1151 |
+
wandb: updating run metadata
|
| 1152 |
+
wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
|
| 1153 |
+
wandb: uploading output.log; uploading wandb-summary.json
|
| 1154 |
+
wandb: uploading history steps 5045-5045, summary, console lines 953-953
|
| 1155 |
+
wandb:
|
| 1156 |
+
wandb: Run history:
|
| 1157 |
+
wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1158 |
+
wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1159 |
+
wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1160 |
+
wandb: eval/loss █▇▇█▆▇▇▆▆▃▂▂▂▂▁▁▁▁▁▁▁▂▂▂▂▂▂▂▂▂▂▂▂▂
|
| 1161 |
+
wandb: eval/perplexity █▇▇█▆▇▇▆▆▃▂▂▂▂▁▁▁▁▁▁▁▂▂▂▂▂▂▂▂▂▂▂▂▂
|
| 1162 |
+
wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1163 |
+
wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1164 |
+
wandb: speed/ms_per_step ▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁
|
| 1165 |
+
wandb: speed/samples_per_sec ██████████████▁█████████████████████████
|
| 1166 |
+
wandb: speed/steps_per_sec ▅▄▄▃▅▄▆▇▄▅▃▃▂▆▅▃▁▆▆▄▄▂▃▂▂▄▂▇▆▇▄▄▄▆█▅▅▄▃▇
|
| 1167 |
+
wandb: +4 ...
|
| 1168 |
+
wandb:
|
| 1169 |
+
wandb: Run summary:
|
| 1170 |
+
wandb: eval/dense_interval_active 0
|
| 1171 |
+
wandb: eval/dense_interval_end -1
|
| 1172 |
+
wandb: eval/dense_interval_start -1
|
| 1173 |
+
wandb: eval/loss 2.49892
|
| 1174 |
+
wandb: eval/perplexity 12.16937
|
| 1175 |
+
wandb: eval/uses_dense_attention 1
|
| 1176 |
+
wandb: eval/uses_sparse_attention 0
|
| 1177 |
+
wandb: speed/ms_per_step 1917.87317
|
| 1178 |
+
wandb: speed/samples_per_sec 31.28465
|
| 1179 |
+
wandb: speed/steps_per_sec 0.52141
|
| 1180 |
+
wandb: +4 ...
|
| 1181 |
+
wandb:
|
| 1182 |
+
wandb: 🚀 View run 1.2b dense 8k seed70 resume41750 6gpu at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 1183 |
+
wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 1184 |
+
wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
|
| 1185 |
+
wandb: Find logs at: ./wandb/run-20260827_022420-rgmkfpnl/logs
|
| 1186 |
+
Training complete!
|
epfl_final/train_logs/p2_1p2b_dense_seed70_6gpu_switch.log
ADDED
|
@@ -0,0 +1,1154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0826 17:29:54.078000 18292 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0826 17:29:54.078000 18292 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0826 17:29:54.078000 18292 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0826 17:29:54.078000 18292 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed70 6gpu switch",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed71"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: False
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250
|
| 73 |
+
Checkpoint activations: False
|
| 74 |
+
Checkpoint activations: False
|
| 75 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
================================================================================
|
| 79 |
+
MODEL ARCHITECTURE
|
| 80 |
+
================================================================================
|
| 81 |
+
Model config:
|
| 82 |
+
vocab_size: 151936
|
| 83 |
+
hidden_size: 1536
|
| 84 |
+
num_hidden_layers: 28
|
| 85 |
+
num_attention_heads: 12
|
| 86 |
+
num_key_value_heads: 4
|
| 87 |
+
head_dim: 128
|
| 88 |
+
intermediate_size: 6144
|
| 89 |
+
max_position_embeddings: 32768
|
| 90 |
+
rope_theta: 1000000
|
| 91 |
+
rope_train_position_scale_set: None
|
| 92 |
+
attention_type: sparse_stochastic
|
| 93 |
+
sparse_block_size: 128
|
| 94 |
+
sparse_num_random_blocks: 31
|
| 95 |
+
sparse_mask_pool_size: 1
|
| 96 |
+
sparse_max_seq_length: 8192
|
| 97 |
+
|
| 98 |
+
Parameter count:
|
| 99 |
+
Total parameters: 1,202,352,640
|
| 100 |
+
Trainable parameters: 1,202,352,640
|
| 101 |
+
Non-trainable parameters: 0
|
| 102 |
+
Model size (BF16): 2.40 GB
|
| 103 |
+
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
LAYER-BY-LAYER SUMMARY
|
| 106 |
+
--------------------------------------------------------------------------------
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 109 |
+
================================================================================================================================================================
|
| 110 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 111 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 112 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 113 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 143 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 144 |
+
================================================================================================================================================================
|
| 145 |
+
Total params: 1,435,726,336
|
| 146 |
+
Trainable params: 1,435,726,336
|
| 147 |
+
Non-trainable params: 0
|
| 148 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 149 |
+
================================================================================================================================================================
|
| 150 |
+
Input size (MB): 0.07
|
| 151 |
+
Forward/backward pass size (MB): 26218.59
|
| 152 |
+
Params size (MB): 2871.45
|
| 153 |
+
Estimated Total Size (MB): 29090.11
|
| 154 |
+
================================================================================================================================================================
|
| 155 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 156 |
+
================================================================================
|
| 157 |
+
|
| 158 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 163 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 164 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 165 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 166 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 167 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 168 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 169 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 170 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 171 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 172 |
+
Dataset split: 3027764 train, 159356 validation
|
| 173 |
+
Training examples (packed, train split): 3027764
|
| 174 |
+
Validation examples (packed, val split): 159356
|
| 175 |
+
Per-device batch size: 1
|
| 176 |
+
World size (GPUs): 6
|
| 177 |
+
Gradient accumulation steps: 10
|
| 178 |
+
Effective batch size (seqs/step): 60
|
| 179 |
+
Sequence length: 8192
|
| 180 |
+
Tokens per optimizer step: 491,520
|
| 181 |
+
Micro-batches per epoch (per rank): 504628
|
| 182 |
+
Optimizer steps per epoch: 50462
|
| 183 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 184 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33250...
|
| 185 |
+
Preserving checkpoint LR state for exact resume: current=[0.00013387516785653213], schedule_base=[0.0005]
|
| 186 |
+
Resumed from checkpoint at step 33250 (epoch 0, micro_step 37500)
|
| 187 |
+
Resuming training from step 33250
|
| 188 |
+
Resuming wandb run rgmkfpnl
|
| 189 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 190 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 191 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 192 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_173107-rgmkfpnl
|
| 193 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 194 |
+
wandb: Resuming run 1.2b dense 8k seed70 6gpu switch
|
| 195 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 196 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/rgmkfpnl
|
| 197 |
+
Starting training...
|
| 198 |
+
Validation at step 33250: Loss = 2.5147, Perplexity = 12.36
|
| 199 |
+
wandb: WARNING Tried to log to step 33250 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 200 |
+
Switching attention mode for step 33251: dense (dense step interval)
|
| 201 |
+
Step 33260/50462 | Loss: 2.5133 | LR: 1.34e-04 | 10270.9ms/step | 5.8 samples/s
|
| 202 |
+
wandb: WARNING Tried to log to step 33260 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 203 |
+
Step 33270/50462 | Loss: 2.4854 | LR: 1.34e-04 | 1885.7ms/step | 31.8 samples/s
|
| 204 |
+
wandb: WARNING Tried to log to step 33270 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 205 |
+
Step 33280/50462 | Loss: 2.4878 | LR: 1.33e-04 | 1892.0ms/step | 31.7 samples/s
|
| 206 |
+
wandb: WARNING Tried to log to step 33280 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 207 |
+
Step 33290/50462 | Loss: 2.5207 | LR: 1.33e-04 | 1898.2ms/step | 31.6 samples/s
|
| 208 |
+
wandb: WARNING Tried to log to step 33290 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 209 |
+
Step 33300/50462 | Loss: 2.5077 | LR: 1.33e-04 | 1901.1ms/step | 31.6 samples/s
|
| 210 |
+
wandb: WARNING Tried to log to step 33300 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 211 |
+
Step 33310/50462 | Loss: 2.5032 | LR: 1.33e-04 | 1906.6ms/step | 31.5 samples/s
|
| 212 |
+
wandb: WARNING Tried to log to step 33310 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 213 |
+
Step 33320/50462 | Loss: 2.4662 | LR: 1.33e-04 | 1908.1ms/step | 31.4 samples/s
|
| 214 |
+
wandb: WARNING Tried to log to step 33320 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Step 33330/50462 | Loss: 2.5206 | LR: 1.33e-04 | 1906.4ms/step | 31.5 samples/s
|
| 216 |
+
wandb: WARNING Tried to log to step 33330 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 217 |
+
Step 33340/50462 | Loss: 2.4841 | LR: 1.33e-04 | 1909.1ms/step | 31.4 samples/s
|
| 218 |
+
wandb: WARNING Tried to log to step 33340 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 219 |
+
Step 33350/50462 | Loss: 2.5607 | LR: 1.32e-04 | 1911.8ms/step | 31.4 samples/s
|
| 220 |
+
wandb: WARNING Tried to log to step 33350 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 221 |
+
Step 33360/50462 | Loss: 2.4827 | LR: 1.32e-04 | 1911.0ms/step | 31.4 samples/s
|
| 222 |
+
wandb: WARNING Tried to log to step 33360 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 223 |
+
Step 33370/50462 | Loss: 2.4959 | LR: 1.32e-04 | 1911.6ms/step | 31.4 samples/s
|
| 224 |
+
wandb: WARNING Tried to log to step 33370 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 225 |
+
Step 33380/50462 | Loss: 2.5044 | LR: 1.32e-04 | 1911.6ms/step | 31.4 samples/s
|
| 226 |
+
wandb: WARNING Tried to log to step 33380 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 227 |
+
Step 33390/50462 | Loss: 2.5119 | LR: 1.32e-04 | 1907.2ms/step | 31.5 samples/s
|
| 228 |
+
wandb: WARNING Tried to log to step 33390 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 229 |
+
Step 33400/50462 | Loss: 2.4696 | LR: 1.32e-04 | 1909.8ms/step | 31.4 samples/s
|
| 230 |
+
wandb: WARNING Tried to log to step 33400 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 231 |
+
Step 33410/50462 | Loss: 2.5027 | LR: 1.32e-04 | 1912.1ms/step | 31.4 samples/s
|
| 232 |
+
wandb: WARNING Tried to log to step 33410 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 233 |
+
Step 33420/50462 | Loss: 2.4818 | LR: 1.32e-04 | 1913.3ms/step | 31.4 samples/s
|
| 234 |
+
wandb: WARNING Tried to log to step 33420 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 235 |
+
Step 33430/50462 | Loss: 2.4731 | LR: 1.31e-04 | 1912.7ms/step | 31.4 samples/s
|
| 236 |
+
wandb: WARNING Tried to log to step 33430 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 237 |
+
Step 33440/50462 | Loss: 2.4966 | LR: 1.31e-04 | 1916.5ms/step | 31.3 samples/s
|
| 238 |
+
wandb: WARNING Tried to log to step 33440 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 239 |
+
Step 33450/50462 | Loss: 2.5432 | LR: 1.31e-04 | 1912.2ms/step | 31.4 samples/s
|
| 240 |
+
wandb: WARNING Tried to log to step 33450 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 241 |
+
Step 33460/50462 | Loss: 2.4893 | LR: 1.31e-04 | 1908.7ms/step | 31.4 samples/s
|
| 242 |
+
wandb: WARNING Tried to log to step 33460 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 243 |
+
Step 33470/50462 | Loss: 2.5461 | LR: 1.31e-04 | 1907.2ms/step | 31.5 samples/s
|
| 244 |
+
wandb: WARNING Tried to log to step 33470 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 245 |
+
Step 33480/50462 | Loss: 2.4717 | LR: 1.31e-04 | 1904.0ms/step | 31.5 samples/s
|
| 246 |
+
wandb: WARNING Tried to log to step 33480 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 247 |
+
Step 33490/50462 | Loss: 2.5124 | LR: 1.31e-04 | 1904.8ms/step | 31.5 samples/s
|
| 248 |
+
wandb: WARNING Tried to log to step 33490 that is less than the current step 33491. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 249 |
+
Step 33500/50462 | Loss: 2.5140 | LR: 1.30e-04 | 1908.0ms/step | 31.4 samples/s
|
| 250 |
+
Validation at step 33500: Loss = 2.5140, Perplexity = 12.35
|
| 251 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33500
|
| 252 |
+
Step 33510/50462 | Loss: 2.5217 | LR: 1.30e-04 | 3288.5ms/step | 18.2 samples/s
|
| 253 |
+
Step 33520/50462 | Loss: 2.5154 | LR: 1.30e-04 | 1905.7ms/step | 31.5 samples/s
|
| 254 |
+
Step 33530/50462 | Loss: 2.4817 | LR: 1.30e-04 | 1910.2ms/step | 31.4 samples/s
|
| 255 |
+
Step 33540/50462 | Loss: 2.5001 | LR: 1.30e-04 | 1912.9ms/step | 31.4 samples/s
|
| 256 |
+
Step 33550/50462 | Loss: 2.5292 | LR: 1.30e-04 | 1914.8ms/step | 31.3 samples/s
|
| 257 |
+
Step 33560/50462 | Loss: 2.5229 | LR: 1.30e-04 | 1912.4ms/step | 31.4 samples/s
|
| 258 |
+
Step 33570/50462 | Loss: 2.4785 | LR: 1.29e-04 | 1912.5ms/step | 31.4 samples/s
|
| 259 |
+
Step 33580/50462 | Loss: 2.5025 | LR: 1.29e-04 | 1911.5ms/step | 31.4 samples/s
|
| 260 |
+
Step 33590/50462 | Loss: 2.5228 | LR: 1.29e-04 | 1913.8ms/step | 31.4 samples/s
|
| 261 |
+
Step 33600/50462 | Loss: 2.5023 | LR: 1.29e-04 | 1917.7ms/step | 31.3 samples/s
|
| 262 |
+
Step 33610/50462 | Loss: 2.5061 | LR: 1.29e-04 | 1911.0ms/step | 31.4 samples/s
|
| 263 |
+
Step 33620/50462 | Loss: 2.4939 | LR: 1.29e-04 | 1911.7ms/step | 31.4 samples/s
|
| 264 |
+
Step 33630/50462 | Loss: 2.4811 | LR: 1.29e-04 | 1912.7ms/step | 31.4 samples/s
|
| 265 |
+
Step 33640/50462 | Loss: 2.4906 | LR: 1.28e-04 | 1913.6ms/step | 31.4 samples/s
|
| 266 |
+
Step 33650/50462 | Loss: 2.4895 | LR: 1.28e-04 | 1916.0ms/step | 31.3 samples/s
|
| 267 |
+
Step 33660/50462 | Loss: 2.5235 | LR: 1.28e-04 | 1917.7ms/step | 31.3 samples/s
|
| 268 |
+
Step 33670/50462 | Loss: 2.4564 | LR: 1.28e-04 | 1917.6ms/step | 31.3 samples/s
|
| 269 |
+
Step 33680/50462 | Loss: 2.4563 | LR: 1.28e-04 | 1919.6ms/step | 31.3 samples/s
|
| 270 |
+
Step 33690/50462 | Loss: 2.5245 | LR: 1.28e-04 | 1922.3ms/step | 31.2 samples/s
|
| 271 |
+
Step 33700/50462 | Loss: 2.5313 | LR: 1.28e-04 | 1918.9ms/step | 31.3 samples/s
|
| 272 |
+
Step 33710/50462 | Loss: 2.5310 | LR: 1.27e-04 | 1913.2ms/step | 31.4 samples/s
|
| 273 |
+
Step 33720/50462 | Loss: 2.4870 | LR: 1.27e-04 | 1911.7ms/step | 31.4 samples/s
|
| 274 |
+
Step 33730/50462 | Loss: 2.4817 | LR: 1.27e-04 | 1912.3ms/step | 31.4 samples/s
|
| 275 |
+
Step 33740/50462 | Loss: 2.5252 | LR: 1.27e-04 | 1909.0ms/step | 31.4 samples/s
|
| 276 |
+
Step 33750/50462 | Loss: 2.4947 | LR: 1.27e-04 | 1908.8ms/step | 31.4 samples/s
|
| 277 |
+
Validation at step 33750: Loss = 2.5121, Perplexity = 12.33
|
| 278 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-33750
|
| 279 |
+
Step 33760/50462 | Loss: 2.4622 | LR: 1.27e-04 | 3246.6ms/step | 18.5 samples/s
|
| 280 |
+
Step 33770/50462 | Loss: 2.5154 | LR: 1.27e-04 | 1904.4ms/step | 31.5 samples/s
|
| 281 |
+
Step 33780/50462 | Loss: 2.4942 | LR: 1.27e-04 | 1908.5ms/step | 31.4 samples/s
|
| 282 |
+
Step 33790/50462 | Loss: 2.5323 | LR: 1.26e-04 | 1907.9ms/step | 31.4 samples/s
|
| 283 |
+
Step 33800/50462 | Loss: 2.5361 | LR: 1.26e-04 | 1909.8ms/step | 31.4 samples/s
|
| 284 |
+
Step 33810/50462 | Loss: 2.4906 | LR: 1.26e-04 | 1915.1ms/step | 31.3 samples/s
|
| 285 |
+
Step 33820/50462 | Loss: 2.5137 | LR: 1.26e-04 | 1914.6ms/step | 31.3 samples/s
|
| 286 |
+
Step 33830/50462 | Loss: 2.5447 | LR: 1.26e-04 | 1919.9ms/step | 31.3 samples/s
|
| 287 |
+
Step 33840/50462 | Loss: 2.5328 | LR: 1.26e-04 | 1909.1ms/step | 31.4 samples/s
|
| 288 |
+
Step 33850/50462 | Loss: 2.4883 | LR: 1.26e-04 | 1914.6ms/step | 31.3 samples/s
|
| 289 |
+
Step 33860/50462 | Loss: 2.4988 | LR: 1.25e-04 | 1916.7ms/step | 31.3 samples/s
|
| 290 |
+
Step 33870/50462 | Loss: 2.4962 | LR: 1.25e-04 | 1914.2ms/step | 31.3 samples/s
|
| 291 |
+
Step 33880/50462 | Loss: 2.4935 | LR: 1.25e-04 | 1919.0ms/step | 31.3 samples/s
|
| 292 |
+
Step 33890/50462 | Loss: 2.4720 | LR: 1.25e-04 | 1909.2ms/step | 31.4 samples/s
|
| 293 |
+
Step 33900/50462 | Loss: 2.4888 | LR: 1.25e-04 | 1904.4ms/step | 31.5 samples/s
|
| 294 |
+
Step 33910/50462 | Loss: 2.5052 | LR: 1.25e-04 | 1908.6ms/step | 31.4 samples/s
|
| 295 |
+
Step 33920/50462 | Loss: 2.4829 | LR: 1.25e-04 | 1914.9ms/step | 31.3 samples/s
|
| 296 |
+
Step 33930/50462 | Loss: 2.4855 | LR: 1.24e-04 | 1917.0ms/step | 31.3 samples/s
|
| 297 |
+
Step 33940/50462 | Loss: 2.4702 | LR: 1.24e-04 | 1916.3ms/step | 31.3 samples/s
|
| 298 |
+
Step 33950/50462 | Loss: 2.5520 | LR: 1.24e-04 | 1917.9ms/step | 31.3 samples/s
|
| 299 |
+
Step 33960/50462 | Loss: 2.5291 | LR: 1.24e-04 | 1915.6ms/step | 31.3 samples/s
|
| 300 |
+
Step 33970/50462 | Loss: 2.5355 | LR: 1.24e-04 | 1910.8ms/step | 31.4 samples/s
|
| 301 |
+
Step 33980/50462 | Loss: 2.4861 | LR: 1.24e-04 | 1907.2ms/step | 31.5 samples/s
|
| 302 |
+
Step 33990/50462 | Loss: 2.5019 | LR: 1.24e-04 | 1915.1ms/step | 31.3 samples/s
|
| 303 |
+
Step 34000/50462 | Loss: 2.5265 | LR: 1.24e-04 | 1915.9ms/step | 31.3 samples/s
|
| 304 |
+
Validation at step 34000: Loss = 2.5119, Perplexity = 12.33
|
| 305 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34000
|
| 306 |
+
Step 34010/50462 | Loss: 2.5237 | LR: 1.23e-04 | 3237.1ms/step | 18.5 samples/s
|
| 307 |
+
Step 34020/50462 | Loss: 2.4873 | LR: 1.23e-04 | 1906.4ms/step | 31.5 samples/s
|
| 308 |
+
Step 34030/50462 | Loss: 2.5294 | LR: 1.23e-04 | 1910.5ms/step | 31.4 samples/s
|
| 309 |
+
Step 34040/50462 | Loss: 2.4993 | LR: 1.23e-04 | 1913.1ms/step | 31.4 samples/s
|
| 310 |
+
Step 34050/50462 | Loss: 2.4872 | LR: 1.23e-04 | 1904.1ms/step | 31.5 samples/s
|
| 311 |
+
Step 34060/50462 | Loss: 2.5320 | LR: 1.23e-04 | 1909.3ms/step | 31.4 samples/s
|
| 312 |
+
Step 34070/50462 | Loss: 2.5202 | LR: 1.23e-04 | 1908.6ms/step | 31.4 samples/s
|
| 313 |
+
Step 34080/50462 | Loss: 2.5193 | LR: 1.22e-04 | 1908.1ms/step | 31.4 samples/s
|
| 314 |
+
Step 34090/50462 | Loss: 2.5235 | LR: 1.22e-04 | 1910.8ms/step | 31.4 samples/s
|
| 315 |
+
Step 34100/50462 | Loss: 2.5209 | LR: 1.22e-04 | 1909.2ms/step | 31.4 samples/s
|
| 316 |
+
Step 34110/50462 | Loss: 2.4700 | LR: 1.22e-04 | 1909.7ms/step | 31.4 samples/s
|
| 317 |
+
Step 34120/50462 | Loss: 2.4972 | LR: 1.22e-04 | 1911.2ms/step | 31.4 samples/s
|
| 318 |
+
Step 34130/50462 | Loss: 2.5175 | LR: 1.22e-04 | 1914.1ms/step | 31.3 samples/s
|
| 319 |
+
Step 34140/50462 | Loss: 2.4870 | LR: 1.22e-04 | 1911.0ms/step | 31.4 samples/s
|
| 320 |
+
Step 34150/50462 | Loss: 2.4972 | LR: 1.21e-04 | 1914.4ms/step | 31.3 samples/s
|
| 321 |
+
Step 34160/50462 | Loss: 2.5286 | LR: 1.21e-04 | 1912.4ms/step | 31.4 samples/s
|
| 322 |
+
Step 34170/50462 | Loss: 2.5389 | LR: 1.21e-04 | 1910.1ms/step | 31.4 samples/s
|
| 323 |
+
Step 34180/50462 | Loss: 2.5542 | LR: 1.21e-04 | 1911.4ms/step | 31.4 samples/s
|
| 324 |
+
Step 34190/50462 | Loss: 2.4777 | LR: 1.21e-04 | 1915.9ms/step | 31.3 samples/s
|
| 325 |
+
Step 34200/50462 | Loss: 2.4670 | LR: 1.21e-04 | 1917.0ms/step | 31.3 samples/s
|
| 326 |
+
Step 34210/50462 | Loss: 2.5238 | LR: 1.21e-04 | 1913.2ms/step | 31.4 samples/s
|
| 327 |
+
Step 34220/50462 | Loss: 2.4937 | LR: 1.21e-04 | 1918.5ms/step | 31.3 samples/s
|
| 328 |
+
Step 34230/50462 | Loss: 2.5384 | LR: 1.20e-04 | 1919.5ms/step | 31.3 samples/s
|
| 329 |
+
Step 34240/50462 | Loss: 2.5210 | LR: 1.20e-04 | 1918.2ms/step | 31.3 samples/s
|
| 330 |
+
Step 34250/50462 | Loss: 2.5202 | LR: 1.20e-04 | 1917.8ms/step | 31.3 samples/s
|
| 331 |
+
Validation at step 34250: Loss = 2.5109, Perplexity = 12.32
|
| 332 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34250
|
| 333 |
+
Step 34260/50462 | Loss: 2.5426 | LR: 1.20e-04 | 3266.0ms/step | 18.4 samples/s
|
| 334 |
+
Step 34270/50462 | Loss: 2.5089 | LR: 1.20e-04 | 1900.6ms/step | 31.6 samples/s
|
| 335 |
+
Step 34280/50462 | Loss: 2.5460 | LR: 1.20e-04 | 1909.8ms/step | 31.4 samples/s
|
| 336 |
+
Step 34290/50462 | Loss: 2.5079 | LR: 1.20e-04 | 1906.5ms/step | 31.5 samples/s
|
| 337 |
+
Step 34300/50462 | Loss: 2.4931 | LR: 1.19e-04 | 1909.6ms/step | 31.4 samples/s
|
| 338 |
+
Step 34310/50462 | Loss: 2.4819 | LR: 1.19e-04 | 1913.5ms/step | 31.4 samples/s
|
| 339 |
+
Step 34320/50462 | Loss: 2.5306 | LR: 1.19e-04 | 1913.1ms/step | 31.4 samples/s
|
| 340 |
+
Step 34330/50462 | Loss: 2.5054 | LR: 1.19e-04 | 1912.7ms/step | 31.4 samples/s
|
| 341 |
+
Step 34340/50462 | Loss: 2.5203 | LR: 1.19e-04 | 1904.2ms/step | 31.5 samples/s
|
| 342 |
+
Step 34350/50462 | Loss: 2.5127 | LR: 1.19e-04 | 1906.6ms/step | 31.5 samples/s
|
| 343 |
+
Step 34360/50462 | Loss: 2.5046 | LR: 1.19e-04 | 1915.5ms/step | 31.3 samples/s
|
| 344 |
+
Step 34370/50462 | Loss: 2.4960 | LR: 1.19e-04 | 1911.9ms/step | 31.4 samples/s
|
| 345 |
+
Step 34380/50462 | Loss: 2.5016 | LR: 1.18e-04 | 1915.1ms/step | 31.3 samples/s
|
| 346 |
+
Step 34390/50462 | Loss: 2.5344 | LR: 1.18e-04 | 1913.9ms/step | 31.3 samples/s
|
| 347 |
+
Step 34400/50462 | Loss: 2.5017 | LR: 1.18e-04 | 1916.5ms/step | 31.3 samples/s
|
| 348 |
+
Step 34410/50462 | Loss: 2.4961 | LR: 1.18e-04 | 1913.2ms/step | 31.4 samples/s
|
| 349 |
+
Step 34420/50462 | Loss: 2.5027 | LR: 1.18e-04 | 1912.9ms/step | 31.4 samples/s
|
| 350 |
+
Step 34430/50462 | Loss: 2.5035 | LR: 1.18e-04 | 1914.8ms/step | 31.3 samples/s
|
| 351 |
+
Step 34440/50462 | Loss: 2.4683 | LR: 1.18e-04 | 1916.8ms/step | 31.3 samples/s
|
| 352 |
+
Step 34450/50462 | Loss: 2.5380 | LR: 1.17e-04 | 1916.9ms/step | 31.3 samples/s
|
| 353 |
+
Step 34460/50462 | Loss: 2.4745 | LR: 1.17e-04 | 1910.0ms/step | 31.4 samples/s
|
| 354 |
+
Step 34470/50462 | Loss: 2.5739 | LR: 1.17e-04 | 1906.9ms/step | 31.5 samples/s
|
| 355 |
+
Step 34480/50462 | Loss: 2.5255 | LR: 1.17e-04 | 1909.8ms/step | 31.4 samples/s
|
| 356 |
+
Step 34490/50462 | Loss: 2.5326 | LR: 1.17e-04 | 1912.1ms/step | 31.4 samples/s
|
| 357 |
+
Step 34500/50462 | Loss: 2.5316 | LR: 1.17e-04 | 1909.1ms/step | 31.4 samples/s
|
| 358 |
+
Validation at step 34500: Loss = 2.5103, Perplexity = 12.31
|
| 359 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34500
|
| 360 |
+
Step 34510/50462 | Loss: 2.4788 | LR: 1.17e-04 | 3224.7ms/step | 18.6 samples/s
|
| 361 |
+
Step 34520/50462 | Loss: 2.5266 | LR: 1.16e-04 | 1903.0ms/step | 31.5 samples/s
|
| 362 |
+
Step 34530/50462 | Loss: 2.5230 | LR: 1.16e-04 | 1908.8ms/step | 31.4 samples/s
|
| 363 |
+
Step 34540/50462 | Loss: 2.5101 | LR: 1.16e-04 | 1914.6ms/step | 31.3 samples/s
|
| 364 |
+
Step 34550/50462 | Loss: 2.4883 | LR: 1.16e-04 | 1912.1ms/step | 31.4 samples/s
|
| 365 |
+
Step 34560/50462 | Loss: 2.4728 | LR: 1.16e-04 | 1912.6ms/step | 31.4 samples/s
|
| 366 |
+
Step 34570/50462 | Loss: 2.5033 | LR: 1.16e-04 | 1916.8ms/step | 31.3 samples/s
|
| 367 |
+
Step 34580/50462 | Loss: 2.5262 | LR: 1.16e-04 | 1917.8ms/step | 31.3 samples/s
|
| 368 |
+
Step 34590/50462 | Loss: 2.5026 | LR: 1.16e-04 | 1919.8ms/step | 31.3 samples/s
|
| 369 |
+
Step 34600/50462 | Loss: 2.5039 | LR: 1.15e-04 | 1916.8ms/step | 31.3 samples/s
|
| 370 |
+
Step 34610/50462 | Loss: 2.5155 | LR: 1.15e-04 | 1917.4ms/step | 31.3 samples/s
|
| 371 |
+
Step 34620/50462 | Loss: 2.4741 | LR: 1.15e-04 | 1916.6ms/step | 31.3 samples/s
|
| 372 |
+
Step 34630/50462 | Loss: 2.5154 | LR: 1.15e-04 | 1921.0ms/step | 31.2 samples/s
|
| 373 |
+
Step 34640/50462 | Loss: 2.4573 | LR: 1.15e-04 | 1919.7ms/step | 31.3 samples/s
|
| 374 |
+
Step 34650/50462 | Loss: 2.5462 | LR: 1.15e-04 | 1919.2ms/step | 31.3 samples/s
|
| 375 |
+
Step 34660/50462 | Loss: 2.5216 | LR: 1.15e-04 | 1919.8ms/step | 31.3 samples/s
|
| 376 |
+
Step 34670/50462 | Loss: 2.4915 | LR: 1.15e-04 | 1919.4ms/step | 31.3 samples/s
|
| 377 |
+
Step 34680/50462 | Loss: 2.5600 | LR: 1.14e-04 | 1915.1ms/step | 31.3 samples/s
|
| 378 |
+
Step 34690/50462 | Loss: 2.4741 | LR: 1.14e-04 | 1914.8ms/step | 31.3 samples/s
|
| 379 |
+
Step 34700/50462 | Loss: 2.4768 | LR: 1.14e-04 | 1917.9ms/step | 31.3 samples/s
|
| 380 |
+
Step 34710/50462 | Loss: 2.5448 | LR: 1.14e-04 | 1919.1ms/step | 31.3 samples/s
|
| 381 |
+
Step 34720/50462 | Loss: 2.5152 | LR: 1.14e-04 | 1919.4ms/step | 31.3 samples/s
|
| 382 |
+
Step 34730/50462 | Loss: 2.4990 | LR: 1.14e-04 | 1917.3ms/step | 31.3 samples/s
|
| 383 |
+
Step 34740/50462 | Loss: 2.5202 | LR: 1.14e-04 | 1925.5ms/step | 31.2 samples/s
|
| 384 |
+
Step 34750/50462 | Loss: 2.4598 | LR: 1.13e-04 | 1920.6ms/step | 31.2 samples/s
|
| 385 |
+
Validation at step 34750: Loss = 2.5094, Perplexity = 12.30
|
| 386 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-34750
|
| 387 |
+
Step 34760/50462 | Loss: 2.5515 | LR: 1.13e-04 | 3237.6ms/step | 18.5 samples/s
|
| 388 |
+
Step 34770/50462 | Loss: 2.4988 | LR: 1.13e-04 | 1905.9ms/step | 31.5 samples/s
|
| 389 |
+
Step 34780/50462 | Loss: 2.5322 | LR: 1.13e-04 | 1906.0ms/step | 31.5 samples/s
|
| 390 |
+
Step 34790/50462 | Loss: 2.5357 | LR: 1.13e-04 | 1913.4ms/step | 31.4 samples/s
|
| 391 |
+
Step 34800/50462 | Loss: 2.5396 | LR: 1.13e-04 | 1911.3ms/step | 31.4 samples/s
|
| 392 |
+
Step 34810/50462 | Loss: 2.5178 | LR: 1.13e-04 | 1910.5ms/step | 31.4 samples/s
|
| 393 |
+
Step 34820/50462 | Loss: 2.5023 | LR: 1.13e-04 | 1914.4ms/step | 31.3 samples/s
|
| 394 |
+
Step 34830/50462 | Loss: 2.5432 | LR: 1.12e-04 | 1918.4ms/step | 31.3 samples/s
|
| 395 |
+
Step 34840/50462 | Loss: 2.4898 | LR: 1.12e-04 | 1918.1ms/step | 31.3 samples/s
|
| 396 |
+
Step 34850/50462 | Loss: 2.5530 | LR: 1.12e-04 | 1919.3ms/step | 31.3 samples/s
|
| 397 |
+
Step 34860/50462 | Loss: 2.5086 | LR: 1.12e-04 | 1915.8ms/step | 31.3 samples/s
|
| 398 |
+
Step 34870/50462 | Loss: 2.4784 | LR: 1.12e-04 | 1919.2ms/step | 31.3 samples/s
|
| 399 |
+
Step 34880/50462 | Loss: 2.4975 | LR: 1.12e-04 | 1919.6ms/step | 31.3 samples/s
|
| 400 |
+
Step 34890/50462 | Loss: 2.5464 | LR: 1.12e-04 | 1920.1ms/step | 31.2 samples/s
|
| 401 |
+
Step 34900/50462 | Loss: 2.4681 | LR: 1.11e-04 | 1920.0ms/step | 31.3 samples/s
|
| 402 |
+
Step 34910/50462 | Loss: 2.5037 | LR: 1.11e-04 | 1920.0ms/step | 31.2 samples/s
|
| 403 |
+
Step 34920/50462 | Loss: 2.5063 | LR: 1.11e-04 | 1919.6ms/step | 31.3 samples/s
|
| 404 |
+
Step 34930/50462 | Loss: 2.4780 | LR: 1.11e-04 | 1920.1ms/step | 31.2 samples/s
|
| 405 |
+
Step 34940/50462 | Loss: 2.5065 | LR: 1.11e-04 | 1920.3ms/step | 31.2 samples/s
|
| 406 |
+
Step 34950/50462 | Loss: 2.5188 | LR: 1.11e-04 | 1921.9ms/step | 31.2 samples/s
|
| 407 |
+
Step 34960/50462 | Loss: 2.5054 | LR: 1.11e-04 | 1920.4ms/step | 31.2 samples/s
|
| 408 |
+
Step 34970/50462 | Loss: 2.4901 | LR: 1.11e-04 | 1919.8ms/step | 31.3 samples/s
|
| 409 |
+
Step 34980/50462 | Loss: 2.5384 | LR: 1.10e-04 | 1915.2ms/step | 31.3 samples/s
|
| 410 |
+
Step 34990/50462 | Loss: 2.4836 | LR: 1.10e-04 | 1908.9ms/step | 31.4 samples/s
|
| 411 |
+
Step 35000/50462 | Loss: 2.5253 | LR: 1.10e-04 | 1910.8ms/step | 31.4 samples/s
|
| 412 |
+
Validation at step 35000: Loss = 2.5090, Perplexity = 12.29
|
| 413 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35000
|
| 414 |
+
Step 35010/50462 | Loss: 2.5122 | LR: 1.10e-04 | 3235.6ms/step | 18.5 samples/s
|
| 415 |
+
Step 35020/50462 | Loss: 2.4542 | LR: 1.10e-04 | 1902.9ms/step | 31.5 samples/s
|
| 416 |
+
Step 35030/50462 | Loss: 2.5578 | LR: 1.10e-04 | 1908.9ms/step | 31.4 samples/s
|
| 417 |
+
Step 35040/50462 | Loss: 2.4686 | LR: 1.10e-04 | 1908.4ms/step | 31.4 samples/s
|
| 418 |
+
Step 35050/50462 | Loss: 2.4992 | LR: 1.09e-04 | 1909.2ms/step | 31.4 samples/s
|
| 419 |
+
Step 35060/50462 | Loss: 2.5092 | LR: 1.09e-04 | 1913.8ms/step | 31.4 samples/s
|
| 420 |
+
Step 35070/50462 | Loss: 2.4695 | LR: 1.09e-04 | 1919.2ms/step | 31.3 samples/s
|
| 421 |
+
Step 35080/50462 | Loss: 2.5148 | LR: 1.09e-04 | 1917.4ms/step | 31.3 samples/s
|
| 422 |
+
Step 35090/50462 | Loss: 2.5370 | LR: 1.09e-04 | 1916.5ms/step | 31.3 samples/s
|
| 423 |
+
Step 35100/50462 | Loss: 2.5033 | LR: 1.09e-04 | 1913.9ms/step | 31.3 samples/s
|
| 424 |
+
Step 35110/50462 | Loss: 2.5233 | LR: 1.09e-04 | 1907.3ms/step | 31.5 samples/s
|
| 425 |
+
Step 35120/50462 | Loss: 2.5232 | LR: 1.09e-04 | 1912.2ms/step | 31.4 samples/s
|
| 426 |
+
Step 35130/50462 | Loss: 2.4695 | LR: 1.08e-04 | 1916.5ms/step | 31.3 samples/s
|
| 427 |
+
Step 35140/50462 | Loss: 2.5140 | LR: 1.08e-04 | 1911.0ms/step | 31.4 samples/s
|
| 428 |
+
Step 35150/50462 | Loss: 2.4806 | LR: 1.08e-04 | 1908.5ms/step | 31.4 samples/s
|
| 429 |
+
Step 35160/50462 | Loss: 2.4722 | LR: 1.08e-04 | 1912.2ms/step | 31.4 samples/s
|
| 430 |
+
Step 35170/50462 | Loss: 2.5323 | LR: 1.08e-04 | 1916.5ms/step | 31.3 samples/s
|
| 431 |
+
Step 35180/50462 | Loss: 2.5086 | LR: 1.08e-04 | 1915.1ms/step | 31.3 samples/s
|
| 432 |
+
Step 35190/50462 | Loss: 2.4892 | LR: 1.08e-04 | 1913.7ms/step | 31.4 samples/s
|
| 433 |
+
Step 35200/50462 | Loss: 2.4761 | LR: 1.08e-04 | 1917.6ms/step | 31.3 samples/s
|
| 434 |
+
Step 35210/50462 | Loss: 2.4793 | LR: 1.07e-04 | 1921.1ms/step | 31.2 samples/s
|
| 435 |
+
Step 35220/50462 | Loss: 2.4702 | LR: 1.07e-04 | 1920.3ms/step | 31.2 samples/s
|
| 436 |
+
Step 35230/50462 | Loss: 2.5000 | LR: 1.07e-04 | 1922.9ms/step | 31.2 samples/s
|
| 437 |
+
Step 35240/50462 | Loss: 2.5110 | LR: 1.07e-04 | 1921.1ms/step | 31.2 samples/s
|
| 438 |
+
Step 35250/50462 | Loss: 2.5049 | LR: 1.07e-04 | 1917.1ms/step | 31.3 samples/s
|
| 439 |
+
Validation at step 35250: Loss = 2.5081, Perplexity = 12.28
|
| 440 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35250
|
| 441 |
+
Step 35260/50462 | Loss: 2.5157 | LR: 1.07e-04 | 3240.8ms/step | 18.5 samples/s
|
| 442 |
+
Step 35270/50462 | Loss: 2.5146 | LR: 1.07e-04 | 1900.4ms/step | 31.6 samples/s
|
| 443 |
+
Step 35280/50462 | Loss: 2.5432 | LR: 1.07e-04 | 1910.4ms/step | 31.4 samples/s
|
| 444 |
+
Step 35290/50462 | Loss: 2.5248 | LR: 1.06e-04 | 1911.1ms/step | 31.4 samples/s
|
| 445 |
+
Step 35300/50462 | Loss: 2.5575 | LR: 1.06e-04 | 1914.6ms/step | 31.3 samples/s
|
| 446 |
+
Step 35310/50462 | Loss: 2.4685 | LR: 1.06e-04 | 1915.4ms/step | 31.3 samples/s
|
| 447 |
+
Step 35320/50462 | Loss: 2.4775 | LR: 1.06e-04 | 1912.5ms/step | 31.4 samples/s
|
| 448 |
+
Step 35330/50462 | Loss: 2.5392 | LR: 1.06e-04 | 1917.5ms/step | 31.3 samples/s
|
| 449 |
+
Step 35340/50462 | Loss: 2.5046 | LR: 1.06e-04 | 1916.1ms/step | 31.3 samples/s
|
| 450 |
+
Step 35350/50462 | Loss: 2.5131 | LR: 1.06e-04 | 1916.4ms/step | 31.3 samples/s
|
| 451 |
+
Step 35360/50462 | Loss: 2.4869 | LR: 1.05e-04 | 1911.7ms/step | 31.4 samples/s
|
| 452 |
+
Step 35370/50462 | Loss: 2.5163 | LR: 1.05e-04 | 1913.1ms/step | 31.4 samples/s
|
| 453 |
+
Step 35380/50462 | Loss: 2.4655 | LR: 1.05e-04 | 1909.3ms/step | 31.4 samples/s
|
| 454 |
+
Step 35390/50462 | Loss: 2.4749 | LR: 1.05e-04 | 1910.0ms/step | 31.4 samples/s
|
| 455 |
+
Step 35400/50462 | Loss: 2.5254 | LR: 1.05e-04 | 1904.6ms/step | 31.5 samples/s
|
| 456 |
+
Step 35410/50462 | Loss: 2.5312 | LR: 1.05e-04 | 1907.1ms/step | 31.5 samples/s
|
| 457 |
+
Step 35420/50462 | Loss: 2.5044 | LR: 1.05e-04 | 1910.2ms/step | 31.4 samples/s
|
| 458 |
+
Step 35430/50462 | Loss: 2.5105 | LR: 1.05e-04 | 1909.6ms/step | 31.4 samples/s
|
| 459 |
+
Step 35440/50462 | Loss: 2.4909 | LR: 1.04e-04 | 1915.3ms/step | 31.3 samples/s
|
| 460 |
+
Step 35450/50462 | Loss: 2.5037 | LR: 1.04e-04 | 1915.3ms/step | 31.3 samples/s
|
| 461 |
+
Step 35460/50462 | Loss: 2.4626 | LR: 1.04e-04 | 1915.4ms/step | 31.3 samples/s
|
| 462 |
+
Step 35470/50462 | Loss: 2.4766 | LR: 1.04e-04 | 1914.7ms/step | 31.3 samples/s
|
| 463 |
+
Step 35480/50462 | Loss: 2.4645 | LR: 1.04e-04 | 1917.5ms/step | 31.3 samples/s
|
| 464 |
+
Step 35490/50462 | Loss: 2.4706 | LR: 1.04e-04 | 1916.9ms/step | 31.3 samples/s
|
| 465 |
+
Step 35500/50462 | Loss: 2.4909 | LR: 1.04e-04 | 1916.1ms/step | 31.3 samples/s
|
| 466 |
+
Validation at step 35500: Loss = 2.5078, Perplexity = 12.28
|
| 467 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35500
|
| 468 |
+
Step 35510/50462 | Loss: 2.5306 | LR: 1.04e-04 | 3252.4ms/step | 18.4 samples/s
|
| 469 |
+
Step 35520/50462 | Loss: 2.4990 | LR: 1.03e-04 | 1897.6ms/step | 31.6 samples/s
|
| 470 |
+
Step 35530/50462 | Loss: 2.5164 | LR: 1.03e-04 | 1904.7ms/step | 31.5 samples/s
|
| 471 |
+
Step 35540/50462 | Loss: 2.5160 | LR: 1.03e-04 | 1907.5ms/step | 31.5 samples/s
|
| 472 |
+
Step 35550/50462 | Loss: 2.5097 | LR: 1.03e-04 | 1910.2ms/step | 31.4 samples/s
|
| 473 |
+
Step 35560/50462 | Loss: 2.4813 | LR: 1.03e-04 | 1911.0ms/step | 31.4 samples/s
|
| 474 |
+
Step 35570/50462 | Loss: 2.5209 | LR: 1.03e-04 | 1909.8ms/step | 31.4 samples/s
|
| 475 |
+
Step 35580/50462 | Loss: 2.4975 | LR: 1.03e-04 | 1908.4ms/step | 31.4 samples/s
|
| 476 |
+
Step 35590/50462 | Loss: 2.5224 | LR: 1.03e-04 | 1911.9ms/step | 31.4 samples/s
|
| 477 |
+
Step 35600/50462 | Loss: 2.4925 | LR: 1.02e-04 | 1911.8ms/step | 31.4 samples/s
|
| 478 |
+
Step 35610/50462 | Loss: 2.4890 | LR: 1.02e-04 | 1911.8ms/step | 31.4 samples/s
|
| 479 |
+
Step 35620/50462 | Loss: 2.4705 | LR: 1.02e-04 | 1906.9ms/step | 31.5 samples/s
|
| 480 |
+
Step 35630/50462 | Loss: 2.4662 | LR: 1.02e-04 | 1911.4ms/step | 31.4 samples/s
|
| 481 |
+
Step 35640/50462 | Loss: 2.4917 | LR: 1.02e-04 | 1914.5ms/step | 31.3 samples/s
|
| 482 |
+
Step 35650/50462 | Loss: 2.4901 | LR: 1.02e-04 | 1915.5ms/step | 31.3 samples/s
|
| 483 |
+
Step 35660/50462 | Loss: 2.5070 | LR: 1.02e-04 | 1918.7ms/step | 31.3 samples/s
|
| 484 |
+
Step 35670/50462 | Loss: 2.5124 | LR: 1.02e-04 | 1919.1ms/step | 31.3 samples/s
|
| 485 |
+
Step 35680/50462 | Loss: 2.5352 | LR: 1.01e-04 | 1916.9ms/step | 31.3 samples/s
|
| 486 |
+
Step 35690/50462 | Loss: 2.5264 | LR: 1.01e-04 | 1910.6ms/step | 31.4 samples/s
|
| 487 |
+
Step 35700/50462 | Loss: 2.5406 | LR: 1.01e-04 | 1912.7ms/step | 31.4 samples/s
|
| 488 |
+
Step 35710/50462 | Loss: 2.4734 | LR: 1.01e-04 | 1919.6ms/step | 31.3 samples/s
|
| 489 |
+
Step 35720/50462 | Loss: 2.4802 | LR: 1.01e-04 | 1920.3ms/step | 31.2 samples/s
|
| 490 |
+
Step 35730/50462 | Loss: 2.4793 | LR: 1.01e-04 | 1919.2ms/step | 31.3 samples/s
|
| 491 |
+
Step 35740/50462 | Loss: 2.5191 | LR: 1.01e-04 | 1921.4ms/step | 31.2 samples/s
|
| 492 |
+
Step 35750/50462 | Loss: 2.5311 | LR: 1.00e-04 | 1914.7ms/step | 31.3 samples/s
|
| 493 |
+
Validation at step 35750: Loss = 2.5075, Perplexity = 12.27
|
| 494 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-35750
|
| 495 |
+
Step 35760/50462 | Loss: 2.5122 | LR: 1.00e-04 | 3254.8ms/step | 18.4 samples/s
|
| 496 |
+
Step 35770/50462 | Loss: 2.5166 | LR: 1.00e-04 | 1905.2ms/step | 31.5 samples/s
|
| 497 |
+
Step 35780/50462 | Loss: 2.5171 | LR: 1.00e-04 | 1905.4ms/step | 31.5 samples/s
|
| 498 |
+
Step 35790/50462 | Loss: 2.5029 | LR: 1.00e-04 | 1910.1ms/step | 31.4 samples/s
|
| 499 |
+
Step 35800/50462 | Loss: 2.4661 | LR: 9.99e-05 | 1915.4ms/step | 31.3 samples/s
|
| 500 |
+
Step 35810/50462 | Loss: 2.4831 | LR: 9.97e-05 | 1914.4ms/step | 31.3 samples/s
|
| 501 |
+
Step 35820/50462 | Loss: 2.5079 | LR: 9.96e-05 | 1909.1ms/step | 31.4 samples/s
|
| 502 |
+
Step 35830/50462 | Loss: 2.4778 | LR: 9.95e-05 | 1909.3ms/step | 31.4 samples/s
|
| 503 |
+
Step 35840/50462 | Loss: 2.4756 | LR: 9.94e-05 | 1904.4ms/step | 31.5 samples/s
|
| 504 |
+
Step 35850/50462 | Loss: 2.5409 | LR: 9.92e-05 | 1905.7ms/step | 31.5 samples/s
|
| 505 |
+
Step 35860/50462 | Loss: 2.5298 | LR: 9.91e-05 | 1909.5ms/step | 31.4 samples/s
|
| 506 |
+
Step 35870/50462 | Loss: 2.4948 | LR: 9.90e-05 | 1913.0ms/step | 31.4 samples/s
|
| 507 |
+
Step 35880/50462 | Loss: 2.4963 | LR: 9.88e-05 | 1913.4ms/step | 31.4 samples/s
|
| 508 |
+
Step 35890/50462 | Loss: 2.4910 | LR: 9.87e-05 | 1910.9ms/step | 31.4 samples/s
|
| 509 |
+
Step 35900/50462 | Loss: 2.4747 | LR: 9.86e-05 | 1910.0ms/step | 31.4 samples/s
|
| 510 |
+
Step 35910/50462 | Loss: 2.4642 | LR: 9.85e-05 | 1915.5ms/step | 31.3 samples/s
|
| 511 |
+
Step 35920/50462 | Loss: 2.5029 | LR: 9.83e-05 | 1915.8ms/step | 31.3 samples/s
|
| 512 |
+
Step 35930/50462 | Loss: 2.4728 | LR: 9.82e-05 | 1913.6ms/step | 31.4 samples/s
|
| 513 |
+
Step 35940/50462 | Loss: 2.4901 | LR: 9.81e-05 | 1914.5ms/step | 31.3 samples/s
|
| 514 |
+
Step 35950/50462 | Loss: 2.4715 | LR: 9.80e-05 | 1914.8ms/step | 31.3 samples/s
|
| 515 |
+
Step 35960/50462 | Loss: 2.4834 | LR: 9.78e-05 | 1920.2ms/step | 31.2 samples/s
|
| 516 |
+
Step 35970/50462 | Loss: 2.5085 | LR: 9.77e-05 | 1908.4ms/step | 31.4 samples/s
|
| 517 |
+
Step 35980/50462 | Loss: 2.4668 | LR: 9.76e-05 | 1907.9ms/step | 31.4 samples/s
|
| 518 |
+
Step 35990/50462 | Loss: 2.5036 | LR: 9.75e-05 | 1911.8ms/step | 31.4 samples/s
|
| 519 |
+
Step 36000/50462 | Loss: 2.4970 | LR: 9.73e-05 | 1910.5ms/step | 31.4 samples/s
|
| 520 |
+
Validation at step 36000: Loss = 2.5069, Perplexity = 12.27
|
| 521 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36000
|
| 522 |
+
Step 36010/50462 | Loss: 2.5395 | LR: 9.72e-05 | 3248.6ms/step | 18.5 samples/s
|
| 523 |
+
Step 36020/50462 | Loss: 2.4988 | LR: 9.71e-05 | 1898.2ms/step | 31.6 samples/s
|
| 524 |
+
Step 36030/50462 | Loss: 2.5221 | LR: 9.70e-05 | 1903.5ms/step | 31.5 samples/s
|
| 525 |
+
Step 36040/50462 | Loss: 2.4976 | LR: 9.68e-05 | 1906.2ms/step | 31.5 samples/s
|
| 526 |
+
Step 36050/50462 | Loss: 2.5068 | LR: 9.67e-05 | 1911.1ms/step | 31.4 samples/s
|
| 527 |
+
Step 36060/50462 | Loss: 2.4272 | LR: 9.66e-05 | 1911.5ms/step | 31.4 samples/s
|
| 528 |
+
Step 36070/50462 | Loss: 2.5186 | LR: 9.65e-05 | 1917.8ms/step | 31.3 samples/s
|
| 529 |
+
Step 36080/50462 | Loss: 2.5109 | LR: 9.63e-05 | 1917.7ms/step | 31.3 samples/s
|
| 530 |
+
Step 36090/50462 | Loss: 2.5127 | LR: 9.62e-05 | 1916.5ms/step | 31.3 samples/s
|
| 531 |
+
Step 36100/50462 | Loss: 2.4825 | LR: 9.61e-05 | 1914.4ms/step | 31.3 samples/s
|
| 532 |
+
Step 36110/50462 | Loss: 2.5098 | LR: 9.60e-05 | 1910.2ms/step | 31.4 samples/s
|
| 533 |
+
Step 36120/50462 | Loss: 2.4785 | LR: 9.58e-05 | 1913.9ms/step | 31.4 samples/s
|
| 534 |
+
Step 36130/50462 | Loss: 2.4871 | LR: 9.57e-05 | 1915.4ms/step | 31.3 samples/s
|
| 535 |
+
Step 36140/50462 | Loss: 2.4980 | LR: 9.56e-05 | 1919.2ms/step | 31.3 samples/s
|
| 536 |
+
Step 36150/50462 | Loss: 2.4715 | LR: 9.55e-05 | 1918.9ms/step | 31.3 samples/s
|
| 537 |
+
Step 36160/50462 | Loss: 2.4628 | LR: 9.53e-05 | 1917.5ms/step | 31.3 samples/s
|
| 538 |
+
Step 36170/50462 | Loss: 2.5018 | LR: 9.52e-05 | 1909.1ms/step | 31.4 samples/s
|
| 539 |
+
Step 36180/50462 | Loss: 2.4930 | LR: 9.51e-05 | 1910.5ms/step | 31.4 samples/s
|
| 540 |
+
Step 36190/50462 | Loss: 2.5179 | LR: 9.50e-05 | 1914.3ms/step | 31.3 samples/s
|
| 541 |
+
Step 36200/50462 | Loss: 2.4832 | LR: 9.49e-05 | 1911.7ms/step | 31.4 samples/s
|
| 542 |
+
Step 36210/50462 | Loss: 2.4520 | LR: 9.47e-05 | 1918.9ms/step | 31.3 samples/s
|
| 543 |
+
Step 36220/50462 | Loss: 2.4946 | LR: 9.46e-05 | 1915.5ms/step | 31.3 samples/s
|
| 544 |
+
Step 36230/50462 | Loss: 2.4994 | LR: 9.45e-05 | 1915.1ms/step | 31.3 samples/s
|
| 545 |
+
Step 36240/50462 | Loss: 2.5369 | LR: 9.44e-05 | 1916.8ms/step | 31.3 samples/s
|
| 546 |
+
Step 36250/50462 | Loss: 2.5093 | LR: 9.42e-05 | 1910.9ms/step | 31.4 samples/s
|
| 547 |
+
Validation at step 36250: Loss = 2.5063, Perplexity = 12.26
|
| 548 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36250
|
| 549 |
+
Step 36260/50462 | Loss: 2.5003 | LR: 9.41e-05 | 3319.0ms/step | 18.1 samples/s
|
| 550 |
+
Step 36270/50462 | Loss: 2.4798 | LR: 9.40e-05 | 1899.9ms/step | 31.6 samples/s
|
| 551 |
+
Step 36280/50462 | Loss: 2.4779 | LR: 9.39e-05 | 1907.3ms/step | 31.5 samples/s
|
| 552 |
+
Step 36290/50462 | Loss: 2.5110 | LR: 9.37e-05 | 1915.1ms/step | 31.3 samples/s
|
| 553 |
+
Step 36300/50462 | Loss: 2.4877 | LR: 9.36e-05 | 1914.7ms/step | 31.3 samples/s
|
| 554 |
+
Step 36310/50462 | Loss: 2.4679 | LR: 9.35e-05 | 1918.9ms/step | 31.3 samples/s
|
| 555 |
+
Step 36320/50462 | Loss: 2.5204 | LR: 9.34e-05 | 1917.0ms/step | 31.3 samples/s
|
| 556 |
+
Step 36330/50462 | Loss: 2.5257 | LR: 9.32e-05 | 1911.8ms/step | 31.4 samples/s
|
| 557 |
+
Step 36340/50462 | Loss: 2.5120 | LR: 9.31e-05 | 1909.6ms/step | 31.4 samples/s
|
| 558 |
+
Step 36350/50462 | Loss: 2.4890 | LR: 9.30e-05 | 1912.2ms/step | 31.4 samples/s
|
| 559 |
+
Step 36360/50462 | Loss: 2.5029 | LR: 9.29e-05 | 1913.3ms/step | 31.4 samples/s
|
| 560 |
+
Step 36370/50462 | Loss: 2.4702 | LR: 9.28e-05 | 1906.1ms/step | 31.5 samples/s
|
| 561 |
+
Step 36380/50462 | Loss: 2.5024 | LR: 9.26e-05 | 1907.8ms/step | 31.4 samples/s
|
| 562 |
+
Step 36390/50462 | Loss: 2.5278 | LR: 9.25e-05 | 1910.7ms/step | 31.4 samples/s
|
| 563 |
+
Step 36400/50462 | Loss: 2.4794 | LR: 9.24e-05 | 1902.5ms/step | 31.5 samples/s
|
| 564 |
+
Step 36410/50462 | Loss: 2.4486 | LR: 9.23e-05 | 1908.2ms/step | 31.4 samples/s
|
| 565 |
+
Step 36420/50462 | Loss: 2.4931 | LR: 9.21e-05 | 1912.0ms/step | 31.4 samples/s
|
| 566 |
+
Step 36430/50462 | Loss: 2.4850 | LR: 9.20e-05 | 1912.5ms/step | 31.4 samples/s
|
| 567 |
+
Step 36440/50462 | Loss: 2.5045 | LR: 9.19e-05 | 1916.1ms/step | 31.3 samples/s
|
| 568 |
+
Step 36450/50462 | Loss: 2.4805 | LR: 9.18e-05 | 1917.4ms/step | 31.3 samples/s
|
| 569 |
+
Step 36460/50462 | Loss: 2.4529 | LR: 9.17e-05 | 1917.9ms/step | 31.3 samples/s
|
| 570 |
+
Step 36470/50462 | Loss: 2.5024 | LR: 9.15e-05 | 1915.7ms/step | 31.3 samples/s
|
| 571 |
+
Step 36480/50462 | Loss: 2.5058 | LR: 9.14e-05 | 1913.8ms/step | 31.4 samples/s
|
| 572 |
+
Step 36490/50462 | Loss: 2.5241 | LR: 9.13e-05 | 1915.4ms/step | 31.3 samples/s
|
| 573 |
+
Step 36500/50462 | Loss: 2.5154 | LR: 9.12e-05 | 1912.3ms/step | 31.4 samples/s
|
| 574 |
+
Validation at step 36500: Loss = 2.5055, Perplexity = 12.25
|
| 575 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36500
|
| 576 |
+
Step 36510/50462 | Loss: 2.4969 | LR: 9.10e-05 | 3271.5ms/step | 18.3 samples/s
|
| 577 |
+
Step 36520/50462 | Loss: 2.4981 | LR: 9.09e-05 | 1902.3ms/step | 31.5 samples/s
|
| 578 |
+
Step 36530/50462 | Loss: 2.5404 | LR: 9.08e-05 | 1908.2ms/step | 31.4 samples/s
|
| 579 |
+
Step 36540/50462 | Loss: 2.4950 | LR: 9.07e-05 | 1910.3ms/step | 31.4 samples/s
|
| 580 |
+
Step 36550/50462 | Loss: 2.4931 | LR: 9.06e-05 | 1914.9ms/step | 31.3 samples/s
|
| 581 |
+
Step 36560/50462 | Loss: 2.5049 | LR: 9.04e-05 | 1916.5ms/step | 31.3 samples/s
|
| 582 |
+
Step 36570/50462 | Loss: 2.5463 | LR: 9.03e-05 | 1916.3ms/step | 31.3 samples/s
|
| 583 |
+
Step 36580/50462 | Loss: 2.5118 | LR: 9.02e-05 | 1914.8ms/step | 31.3 samples/s
|
| 584 |
+
Step 36590/50462 | Loss: 2.4822 | LR: 9.01e-05 | 1918.1ms/step | 31.3 samples/s
|
| 585 |
+
Step 36600/50462 | Loss: 2.4988 | LR: 8.99e-05 | 1918.8ms/step | 31.3 samples/s
|
| 586 |
+
Step 36610/50462 | Loss: 2.5313 | LR: 8.98e-05 | 1913.1ms/step | 31.4 samples/s
|
| 587 |
+
Step 36620/50462 | Loss: 2.4485 | LR: 8.97e-05 | 1913.4ms/step | 31.4 samples/s
|
| 588 |
+
Step 36630/50462 | Loss: 2.4852 | LR: 8.96e-05 | 1914.8ms/step | 31.3 samples/s
|
| 589 |
+
Step 36640/50462 | Loss: 2.4604 | LR: 8.95e-05 | 1914.3ms/step | 31.3 samples/s
|
| 590 |
+
Step 36650/50462 | Loss: 2.4901 | LR: 8.93e-05 | 1911.9ms/step | 31.4 samples/s
|
| 591 |
+
Step 36660/50462 | Loss: 2.4532 | LR: 8.92e-05 | 1906.4ms/step | 31.5 samples/s
|
| 592 |
+
Step 36670/50462 | Loss: 2.4870 | LR: 8.91e-05 | 1906.8ms/step | 31.5 samples/s
|
| 593 |
+
Step 36680/50462 | Loss: 2.4976 | LR: 8.90e-05 | 1911.7ms/step | 31.4 samples/s
|
| 594 |
+
Step 36690/50462 | Loss: 2.4430 | LR: 8.89e-05 | 1914.3ms/step | 31.3 samples/s
|
| 595 |
+
Step 36700/50462 | Loss: 2.4809 | LR: 8.87e-05 | 1916.1ms/step | 31.3 samples/s
|
| 596 |
+
Step 36710/50462 | Loss: 2.4622 | LR: 8.86e-05 | 1918.1ms/step | 31.3 samples/s
|
| 597 |
+
Step 36720/50462 | Loss: 2.5381 | LR: 8.85e-05 | 1915.7ms/step | 31.3 samples/s
|
| 598 |
+
Step 36730/50462 | Loss: 2.4593 | LR: 8.84e-05 | 1917.8ms/step | 31.3 samples/s
|
| 599 |
+
Step 36740/50462 | Loss: 2.4952 | LR: 8.83e-05 | 1920.3ms/step | 31.2 samples/s
|
| 600 |
+
Step 36750/50462 | Loss: 2.5218 | LR: 8.81e-05 | 1918.1ms/step | 31.3 samples/s
|
| 601 |
+
Validation at step 36750: Loss = 2.5052, Perplexity = 12.25
|
| 602 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-36750
|
| 603 |
+
Step 36760/50462 | Loss: 2.4895 | LR: 8.80e-05 | 3219.3ms/step | 18.6 samples/s
|
| 604 |
+
Step 36770/50462 | Loss: 2.4900 | LR: 8.79e-05 | 1898.4ms/step | 31.6 samples/s
|
| 605 |
+
Step 36780/50462 | Loss: 2.4407 | LR: 8.78e-05 | 1902.0ms/step | 31.5 samples/s
|
| 606 |
+
Step 36790/50462 | Loss: 2.4694 | LR: 8.77e-05 | 1906.0ms/step | 31.5 samples/s
|
| 607 |
+
Step 36800/50462 | Loss: 2.5387 | LR: 8.75e-05 | 1909.5ms/step | 31.4 samples/s
|
| 608 |
+
Step 36810/50462 | Loss: 2.4934 | LR: 8.74e-05 | 1913.2ms/step | 31.4 samples/s
|
| 609 |
+
Step 36820/50462 | Loss: 2.4705 | LR: 8.73e-05 | 1912.1ms/step | 31.4 samples/s
|
| 610 |
+
Step 36830/50462 | Loss: 2.4749 | LR: 8.72e-05 | 1914.1ms/step | 31.3 samples/s
|
| 611 |
+
Step 36840/50462 | Loss: 2.4848 | LR: 8.71e-05 | 1915.1ms/step | 31.3 samples/s
|
| 612 |
+
Step 36850/50462 | Loss: 2.4981 | LR: 8.69e-05 | 1916.2ms/step | 31.3 samples/s
|
| 613 |
+
Step 36860/50462 | Loss: 2.5250 | LR: 8.68e-05 | 1916.9ms/step | 31.3 samples/s
|
| 614 |
+
Step 36870/50462 | Loss: 2.4698 | LR: 8.67e-05 | 1921.6ms/step | 31.2 samples/s
|
| 615 |
+
Step 36880/50462 | Loss: 2.5026 | LR: 8.66e-05 | 1915.5ms/step | 31.3 samples/s
|
| 616 |
+
Step 36890/50462 | Loss: 2.5003 | LR: 8.65e-05 | 1917.5ms/step | 31.3 samples/s
|
| 617 |
+
Step 36900/50462 | Loss: 2.5101 | LR: 8.63e-05 | 1916.3ms/step | 31.3 samples/s
|
| 618 |
+
Step 36910/50462 | Loss: 2.4833 | LR: 8.62e-05 | 1916.2ms/step | 31.3 samples/s
|
| 619 |
+
Step 36920/50462 | Loss: 2.5346 | LR: 8.61e-05 | 1914.7ms/step | 31.3 samples/s
|
| 620 |
+
Step 36930/50462 | Loss: 2.4820 | LR: 8.60e-05 | 1915.7ms/step | 31.3 samples/s
|
| 621 |
+
Step 36940/50462 | Loss: 2.5242 | LR: 8.59e-05 | 1914.5ms/step | 31.3 samples/s
|
| 622 |
+
Step 36950/50462 | Loss: 2.4873 | LR: 8.57e-05 | 1913.9ms/step | 31.4 samples/s
|
| 623 |
+
Step 36960/50462 | Loss: 2.4774 | LR: 8.56e-05 | 1918.3ms/step | 31.3 samples/s
|
| 624 |
+
Step 36970/50462 | Loss: 2.5293 | LR: 8.55e-05 | 1917.5ms/step | 31.3 samples/s
|
| 625 |
+
Step 36980/50462 | Loss: 2.4873 | LR: 8.54e-05 | 1917.4ms/step | 31.3 samples/s
|
| 626 |
+
Step 36990/50462 | Loss: 2.4967 | LR: 8.53e-05 | 1918.0ms/step | 31.3 samples/s
|
| 627 |
+
Step 37000/50462 | Loss: 2.4924 | LR: 8.51e-05 | 1914.4ms/step | 31.3 samples/s
|
| 628 |
+
Validation at step 37000: Loss = 2.5043, Perplexity = 12.24
|
| 629 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37000
|
| 630 |
+
Step 37010/50462 | Loss: 2.4209 | LR: 8.50e-05 | 3256.8ms/step | 18.4 samples/s
|
| 631 |
+
Step 37020/50462 | Loss: 2.4980 | LR: 8.49e-05 | 1901.4ms/step | 31.6 samples/s
|
| 632 |
+
Step 37030/50462 | Loss: 2.4732 | LR: 8.48e-05 | 1905.7ms/step | 31.5 samples/s
|
| 633 |
+
Step 37040/50462 | Loss: 2.4640 | LR: 8.47e-05 | 1906.6ms/step | 31.5 samples/s
|
| 634 |
+
Step 37050/50462 | Loss: 2.5106 | LR: 8.46e-05 | 1910.0ms/step | 31.4 samples/s
|
| 635 |
+
Step 37060/50462 | Loss: 2.5001 | LR: 8.44e-05 | 1913.4ms/step | 31.4 samples/s
|
| 636 |
+
Step 37070/50462 | Loss: 2.4775 | LR: 8.43e-05 | 1912.4ms/step | 31.4 samples/s
|
| 637 |
+
Step 37080/50462 | Loss: 2.5455 | LR: 8.42e-05 | 1914.3ms/step | 31.3 samples/s
|
| 638 |
+
Step 37090/50462 | Loss: 2.4927 | LR: 8.41e-05 | 1910.2ms/step | 31.4 samples/s
|
| 639 |
+
Step 37100/50462 | Loss: 2.5025 | LR: 8.40e-05 | 1910.6ms/step | 31.4 samples/s
|
| 640 |
+
Step 37110/50462 | Loss: 2.5176 | LR: 8.38e-05 | 1910.0ms/step | 31.4 samples/s
|
| 641 |
+
Step 37120/50462 | Loss: 2.4841 | LR: 8.37e-05 | 1909.9ms/step | 31.4 samples/s
|
| 642 |
+
Step 37130/50462 | Loss: 2.4609 | LR: 8.36e-05 | 1912.9ms/step | 31.4 samples/s
|
| 643 |
+
Step 37140/50462 | Loss: 2.5067 | LR: 8.35e-05 | 1917.8ms/step | 31.3 samples/s
|
| 644 |
+
Step 37150/50462 | Loss: 2.4832 | LR: 8.34e-05 | 1918.3ms/step | 31.3 samples/s
|
| 645 |
+
Step 37160/50462 | Loss: 2.4770 | LR: 8.33e-05 | 1913.7ms/step | 31.4 samples/s
|
| 646 |
+
Step 37170/50462 | Loss: 2.5160 | LR: 8.31e-05 | 1915.2ms/step | 31.3 samples/s
|
| 647 |
+
Step 37180/50462 | Loss: 2.4643 | LR: 8.30e-05 | 1912.6ms/step | 31.4 samples/s
|
| 648 |
+
Step 37190/50462 | Loss: 2.5082 | LR: 8.29e-05 | 1914.6ms/step | 31.3 samples/s
|
| 649 |
+
Step 37200/50462 | Loss: 2.4941 | LR: 8.28e-05 | 1913.7ms/step | 31.4 samples/s
|
| 650 |
+
Step 37210/50462 | Loss: 2.4908 | LR: 8.27e-05 | 1915.0ms/step | 31.3 samples/s
|
| 651 |
+
Step 37220/50462 | Loss: 2.4991 | LR: 8.25e-05 | 1916.7ms/step | 31.3 samples/s
|
| 652 |
+
Step 37230/50462 | Loss: 2.4885 | LR: 8.24e-05 | 1912.5ms/step | 31.4 samples/s
|
| 653 |
+
Step 37240/50462 | Loss: 2.4869 | LR: 8.23e-05 | 1911.1ms/step | 31.4 samples/s
|
| 654 |
+
Step 37250/50462 | Loss: 2.5236 | LR: 8.22e-05 | 1913.5ms/step | 31.4 samples/s
|
| 655 |
+
Validation at step 37250: Loss = 2.5042, Perplexity = 12.23
|
| 656 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37250
|
| 657 |
+
Step 37260/50462 | Loss: 2.4648 | LR: 8.21e-05 | 3232.1ms/step | 18.6 samples/s
|
| 658 |
+
Step 37270/50462 | Loss: 2.5040 | LR: 8.20e-05 | 1901.3ms/step | 31.6 samples/s
|
| 659 |
+
Step 37280/50462 | Loss: 2.5210 | LR: 8.18e-05 | 1906.1ms/step | 31.5 samples/s
|
| 660 |
+
Step 37290/50462 | Loss: 2.5204 | LR: 8.17e-05 | 1909.9ms/step | 31.4 samples/s
|
| 661 |
+
Step 37300/50462 | Loss: 2.4694 | LR: 8.16e-05 | 1912.7ms/step | 31.4 samples/s
|
| 662 |
+
Step 37310/50462 | Loss: 2.5182 | LR: 8.15e-05 | 1914.2ms/step | 31.3 samples/s
|
| 663 |
+
Step 37320/50462 | Loss: 2.4857 | LR: 8.14e-05 | 1915.0ms/step | 31.3 samples/s
|
| 664 |
+
Step 37330/50462 | Loss: 2.4962 | LR: 8.13e-05 | 1911.5ms/step | 31.4 samples/s
|
| 665 |
+
Step 37340/50462 | Loss: 2.5260 | LR: 8.11e-05 | 1915.6ms/step | 31.3 samples/s
|
| 666 |
+
Step 37350/50462 | Loss: 2.5239 | LR: 8.10e-05 | 1918.9ms/step | 31.3 samples/s
|
| 667 |
+
Step 37360/50462 | Loss: 2.5150 | LR: 8.09e-05 | 1915.1ms/step | 31.3 samples/s
|
| 668 |
+
Step 37370/50462 | Loss: 2.5062 | LR: 8.08e-05 | 1915.2ms/step | 31.3 samples/s
|
| 669 |
+
Step 37380/50462 | Loss: 2.4522 | LR: 8.07e-05 | 1912.8ms/step | 31.4 samples/s
|
| 670 |
+
Step 37390/50462 | Loss: 2.4689 | LR: 8.06e-05 | 1917.0ms/step | 31.3 samples/s
|
| 671 |
+
Step 37400/50462 | Loss: 2.4961 | LR: 8.04e-05 | 1915.3ms/step | 31.3 samples/s
|
| 672 |
+
Step 37410/50462 | Loss: 2.5061 | LR: 8.03e-05 | 1918.5ms/step | 31.3 samples/s
|
| 673 |
+
Step 37420/50462 | Loss: 2.5010 | LR: 8.02e-05 | 1915.0ms/step | 31.3 samples/s
|
| 674 |
+
Step 37430/50462 | Loss: 2.5070 | LR: 8.01e-05 | 1916.1ms/step | 31.3 samples/s
|
| 675 |
+
Step 37440/50462 | Loss: 2.4826 | LR: 8.00e-05 | 1916.1ms/step | 31.3 samples/s
|
| 676 |
+
Step 37450/50462 | Loss: 2.4965 | LR: 7.99e-05 | 1919.1ms/step | 31.3 samples/s
|
| 677 |
+
Step 37460/50462 | Loss: 2.5015 | LR: 7.98e-05 | 1918.1ms/step | 31.3 samples/s
|
| 678 |
+
Step 37470/50462 | Loss: 2.4803 | LR: 7.96e-05 | 1919.0ms/step | 31.3 samples/s
|
| 679 |
+
Step 37480/50462 | Loss: 2.4591 | LR: 7.95e-05 | 1917.2ms/step | 31.3 samples/s
|
| 680 |
+
Step 37490/50462 | Loss: 2.4755 | LR: 7.94e-05 | 1917.8ms/step | 31.3 samples/s
|
| 681 |
+
Step 37500/50462 | Loss: 2.4302 | LR: 7.93e-05 | 1916.9ms/step | 31.3 samples/s
|
| 682 |
+
Validation at step 37500: Loss = 2.5035, Perplexity = 12.23
|
| 683 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37500
|
| 684 |
+
Step 37510/50462 | Loss: 2.4933 | LR: 7.92e-05 | 3222.2ms/step | 18.6 samples/s
|
| 685 |
+
Step 37520/50462 | Loss: 2.4861 | LR: 7.91e-05 | 1899.0ms/step | 31.6 samples/s
|
| 686 |
+
Step 37530/50462 | Loss: 2.4549 | LR: 7.89e-05 | 1904.6ms/step | 31.5 samples/s
|
| 687 |
+
Step 37540/50462 | Loss: 2.4372 | LR: 7.88e-05 | 1905.8ms/step | 31.5 samples/s
|
| 688 |
+
Step 37550/50462 | Loss: 2.4803 | LR: 7.87e-05 | 1908.1ms/step | 31.4 samples/s
|
| 689 |
+
Step 37560/50462 | Loss: 2.5232 | LR: 7.86e-05 | 1908.4ms/step | 31.4 samples/s
|
| 690 |
+
Step 37570/50462 | Loss: 2.4755 | LR: 7.85e-05 | 1912.7ms/step | 31.4 samples/s
|
| 691 |
+
Step 37580/50462 | Loss: 2.5029 | LR: 7.84e-05 | 1910.7ms/step | 31.4 samples/s
|
| 692 |
+
Step 37590/50462 | Loss: 2.5202 | LR: 7.83e-05 | 1908.7ms/step | 31.4 samples/s
|
| 693 |
+
Step 37600/50462 | Loss: 2.4905 | LR: 7.81e-05 | 1908.6ms/step | 31.4 samples/s
|
| 694 |
+
Step 37610/50462 | Loss: 2.4892 | LR: 7.80e-05 | 1909.8ms/step | 31.4 samples/s
|
| 695 |
+
Step 37620/50462 | Loss: 2.5195 | LR: 7.79e-05 | 1912.2ms/step | 31.4 samples/s
|
| 696 |
+
Step 37630/50462 | Loss: 2.5068 | LR: 7.78e-05 | 1913.7ms/step | 31.4 samples/s
|
| 697 |
+
Step 37640/50462 | Loss: 2.5121 | LR: 7.77e-05 | 1915.7ms/step | 31.3 samples/s
|
| 698 |
+
Step 37650/50462 | Loss: 2.4986 | LR: 7.76e-05 | 1914.8ms/step | 31.3 samples/s
|
| 699 |
+
Step 37660/50462 | Loss: 2.4897 | LR: 7.75e-05 | 1913.2ms/step | 31.4 samples/s
|
| 700 |
+
Step 37670/50462 | Loss: 2.4722 | LR: 7.73e-05 | 1913.8ms/step | 31.4 samples/s
|
| 701 |
+
Step 37680/50462 | Loss: 2.4942 | LR: 7.72e-05 | 1913.1ms/step | 31.4 samples/s
|
| 702 |
+
Step 37690/50462 | Loss: 2.5142 | LR: 7.71e-05 | 1916.9ms/step | 31.3 samples/s
|
| 703 |
+
Step 37700/50462 | Loss: 2.4717 | LR: 7.70e-05 | 1913.1ms/step | 31.4 samples/s
|
| 704 |
+
Step 37710/50462 | Loss: 2.5051 | LR: 7.69e-05 | 1915.2ms/step | 31.3 samples/s
|
| 705 |
+
Step 37720/50462 | Loss: 2.4899 | LR: 7.68e-05 | 1912.3ms/step | 31.4 samples/s
|
| 706 |
+
Step 37730/50462 | Loss: 2.5075 | LR: 7.67e-05 | 1913.1ms/step | 31.4 samples/s
|
| 707 |
+
Step 37740/50462 | Loss: 2.5137 | LR: 7.65e-05 | 1910.8ms/step | 31.4 samples/s
|
| 708 |
+
Step 37750/50462 | Loss: 2.4850 | LR: 7.64e-05 | 1911.7ms/step | 31.4 samples/s
|
| 709 |
+
Validation at step 37750: Loss = 2.5031, Perplexity = 12.22
|
| 710 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-37750
|
| 711 |
+
Step 37760/50462 | Loss: 2.4698 | LR: 7.63e-05 | 3277.7ms/step | 18.3 samples/s
|
| 712 |
+
Step 37770/50462 | Loss: 2.5251 | LR: 7.62e-05 | 1900.2ms/step | 31.6 samples/s
|
| 713 |
+
Step 37780/50462 | Loss: 2.4860 | LR: 7.61e-05 | 1910.4ms/step | 31.4 samples/s
|
| 714 |
+
Step 37790/50462 | Loss: 2.4662 | LR: 7.60e-05 | 1911.3ms/step | 31.4 samples/s
|
| 715 |
+
Step 37800/50462 | Loss: 2.4280 | LR: 7.59e-05 | 1908.4ms/step | 31.4 samples/s
|
| 716 |
+
Step 37810/50462 | Loss: 2.4986 | LR: 7.57e-05 | 1912.4ms/step | 31.4 samples/s
|
| 717 |
+
Step 37820/50462 | Loss: 2.4835 | LR: 7.56e-05 | 1913.3ms/step | 31.4 samples/s
|
| 718 |
+
Step 37830/50462 | Loss: 2.4961 | LR: 7.55e-05 | 1912.6ms/step | 31.4 samples/s
|
| 719 |
+
Step 37840/50462 | Loss: 2.5270 | LR: 7.54e-05 | 1910.1ms/step | 31.4 samples/s
|
| 720 |
+
Step 37850/50462 | Loss: 2.4732 | LR: 7.53e-05 | 1911.5ms/step | 31.4 samples/s
|
| 721 |
+
Step 37860/50462 | Loss: 2.5476 | LR: 7.52e-05 | 1915.6ms/step | 31.3 samples/s
|
| 722 |
+
Step 37870/50462 | Loss: 2.5253 | LR: 7.51e-05 | 1914.0ms/step | 31.3 samples/s
|
| 723 |
+
Step 37880/50462 | Loss: 2.4975 | LR: 7.50e-05 | 1913.8ms/step | 31.4 samples/s
|
| 724 |
+
Step 37890/50462 | Loss: 2.4717 | LR: 7.48e-05 | 1916.8ms/step | 31.3 samples/s
|
| 725 |
+
Step 37900/50462 | Loss: 2.4347 | LR: 7.47e-05 | 1912.7ms/step | 31.4 samples/s
|
| 726 |
+
Step 37910/50462 | Loss: 2.4776 | LR: 7.46e-05 | 1906.4ms/step | 31.5 samples/s
|
| 727 |
+
Step 37920/50462 | Loss: 2.4940 | LR: 7.45e-05 | 1911.3ms/step | 31.4 samples/s
|
| 728 |
+
Step 37930/50462 | Loss: 2.4999 | LR: 7.44e-05 | 1911.7ms/step | 31.4 samples/s
|
| 729 |
+
Step 37940/50462 | Loss: 2.5382 | LR: 7.43e-05 | 1912.5ms/step | 31.4 samples/s
|
| 730 |
+
Step 37950/50462 | Loss: 2.5078 | LR: 7.42e-05 | 1908.2ms/step | 31.4 samples/s
|
| 731 |
+
Step 37960/50462 | Loss: 2.4972 | LR: 7.41e-05 | 1908.4ms/step | 31.4 samples/s
|
| 732 |
+
Step 37970/50462 | Loss: 2.5354 | LR: 7.39e-05 | 1910.5ms/step | 31.4 samples/s
|
| 733 |
+
Step 37980/50462 | Loss: 2.5073 | LR: 7.38e-05 | 1911.8ms/step | 31.4 samples/s
|
| 734 |
+
Step 37990/50462 | Loss: 2.4999 | LR: 7.37e-05 | 1911.2ms/step | 31.4 samples/s
|
| 735 |
+
Step 38000/50462 | Loss: 2.4788 | LR: 7.36e-05 | 1908.9ms/step | 31.4 samples/s
|
| 736 |
+
Validation at step 38000: Loss = 2.5028, Perplexity = 12.22
|
| 737 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38000
|
| 738 |
+
Step 38010/50462 | Loss: 2.4934 | LR: 7.35e-05 | 3243.9ms/step | 18.5 samples/s
|
| 739 |
+
Step 38020/50462 | Loss: 2.4907 | LR: 7.34e-05 | 1900.6ms/step | 31.6 samples/s
|
| 740 |
+
Step 38030/50462 | Loss: 2.5166 | LR: 7.33e-05 | 1905.4ms/step | 31.5 samples/s
|
| 741 |
+
Step 38040/50462 | Loss: 2.4852 | LR: 7.32e-05 | 1909.2ms/step | 31.4 samples/s
|
| 742 |
+
Step 38050/50462 | Loss: 2.4741 | LR: 7.30e-05 | 1909.6ms/step | 31.4 samples/s
|
| 743 |
+
Step 38060/50462 | Loss: 2.4731 | LR: 7.29e-05 | 1913.2ms/step | 31.4 samples/s
|
| 744 |
+
Step 38070/50462 | Loss: 2.4844 | LR: 7.28e-05 | 1914.3ms/step | 31.3 samples/s
|
| 745 |
+
Step 38080/50462 | Loss: 2.5107 | LR: 7.27e-05 | 1913.7ms/step | 31.4 samples/s
|
| 746 |
+
Step 38090/50462 | Loss: 2.4803 | LR: 7.26e-05 | 1909.9ms/step | 31.4 samples/s
|
| 747 |
+
Step 38100/50462 | Loss: 2.5166 | LR: 7.25e-05 | 1906.1ms/step | 31.5 samples/s
|
| 748 |
+
Step 38110/50462 | Loss: 2.5122 | LR: 7.24e-05 | 1905.0ms/step | 31.5 samples/s
|
| 749 |
+
Step 38120/50462 | Loss: 2.5078 | LR: 7.23e-05 | 1906.2ms/step | 31.5 samples/s
|
| 750 |
+
Step 38130/50462 | Loss: 2.4915 | LR: 7.22e-05 | 1910.9ms/step | 31.4 samples/s
|
| 751 |
+
Step 38140/50462 | Loss: 2.5109 | LR: 7.20e-05 | 1909.8ms/step | 31.4 samples/s
|
| 752 |
+
Step 38150/50462 | Loss: 2.5031 | LR: 7.19e-05 | 1912.6ms/step | 31.4 samples/s
|
| 753 |
+
Step 38160/50462 | Loss: 2.5020 | LR: 7.18e-05 | 1914.3ms/step | 31.3 samples/s
|
| 754 |
+
Step 38170/50462 | Loss: 2.5450 | LR: 7.17e-05 | 1913.1ms/step | 31.4 samples/s
|
| 755 |
+
Step 38180/50462 | Loss: 2.4759 | LR: 7.16e-05 | 1909.0ms/step | 31.4 samples/s
|
| 756 |
+
Step 38190/50462 | Loss: 2.4822 | LR: 7.15e-05 | 1904.3ms/step | 31.5 samples/s
|
| 757 |
+
Step 38200/50462 | Loss: 2.5139 | LR: 7.14e-05 | 1905.1ms/step | 31.5 samples/s
|
| 758 |
+
Step 38210/50462 | Loss: 2.4812 | LR: 7.13e-05 | 1908.6ms/step | 31.4 samples/s
|
| 759 |
+
Step 38220/50462 | Loss: 2.4836 | LR: 7.12e-05 | 1911.8ms/step | 31.4 samples/s
|
| 760 |
+
Step 38230/50462 | Loss: 2.5066 | LR: 7.10e-05 | 1913.9ms/step | 31.3 samples/s
|
| 761 |
+
Step 38240/50462 | Loss: 2.4901 | LR: 7.09e-05 | 1916.0ms/step | 31.3 samples/s
|
| 762 |
+
Step 38250/50462 | Loss: 2.5418 | LR: 7.08e-05 | 1915.8ms/step | 31.3 samples/s
|
| 763 |
+
Validation at step 38250: Loss = 2.5024, Perplexity = 12.21
|
| 764 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38250
|
| 765 |
+
Step 38260/50462 | Loss: 2.4272 | LR: 7.07e-05 | 3235.6ms/step | 18.5 samples/s
|
| 766 |
+
Step 38270/50462 | Loss: 2.4546 | LR: 7.06e-05 | 1902.0ms/step | 31.5 samples/s
|
| 767 |
+
Step 38280/50462 | Loss: 2.5088 | LR: 7.05e-05 | 1907.4ms/step | 31.5 samples/s
|
| 768 |
+
Step 38290/50462 | Loss: 2.4620 | LR: 7.04e-05 | 1908.4ms/step | 31.4 samples/s
|
| 769 |
+
Step 38300/50462 | Loss: 2.5093 | LR: 7.03e-05 | 1911.5ms/step | 31.4 samples/s
|
| 770 |
+
Step 38310/50462 | Loss: 2.5016 | LR: 7.02e-05 | 1908.0ms/step | 31.4 samples/s
|
| 771 |
+
Step 38320/50462 | Loss: 2.4870 | LR: 7.01e-05 | 1907.1ms/step | 31.5 samples/s
|
| 772 |
+
Step 38330/50462 | Loss: 2.4832 | LR: 6.99e-05 | 1906.5ms/step | 31.5 samples/s
|
| 773 |
+
Step 38340/50462 | Loss: 2.4758 | LR: 6.98e-05 | 1914.4ms/step | 31.3 samples/s
|
| 774 |
+
Step 38350/50462 | Loss: 2.5057 | LR: 6.97e-05 | 1914.2ms/step | 31.3 samples/s
|
| 775 |
+
Step 38360/50462 | Loss: 2.4924 | LR: 6.96e-05 | 1912.9ms/step | 31.4 samples/s
|
| 776 |
+
Step 38370/50462 | Loss: 2.4918 | LR: 6.95e-05 | 1914.9ms/step | 31.3 samples/s
|
| 777 |
+
Step 38380/50462 | Loss: 2.4599 | LR: 6.94e-05 | 1908.4ms/step | 31.4 samples/s
|
| 778 |
+
Step 38390/50462 | Loss: 2.4817 | LR: 6.93e-05 | 1907.0ms/step | 31.5 samples/s
|
| 779 |
+
Step 38400/50462 | Loss: 2.5070 | LR: 6.92e-05 | 1909.1ms/step | 31.4 samples/s
|
| 780 |
+
Step 38410/50462 | Loss: 2.5230 | LR: 6.91e-05 | 1904.9ms/step | 31.5 samples/s
|
| 781 |
+
Step 38420/50462 | Loss: 2.4646 | LR: 6.90e-05 | 1901.9ms/step | 31.5 samples/s
|
| 782 |
+
Step 38430/50462 | Loss: 2.4610 | LR: 6.89e-05 | 1906.2ms/step | 31.5 samples/s
|
| 783 |
+
Step 38440/50462 | Loss: 2.4784 | LR: 6.87e-05 | 1910.5ms/step | 31.4 samples/s
|
| 784 |
+
Step 38450/50462 | Loss: 2.4733 | LR: 6.86e-05 | 1910.3ms/step | 31.4 samples/s
|
| 785 |
+
Step 38460/50462 | Loss: 2.4964 | LR: 6.85e-05 | 1903.2ms/step | 31.5 samples/s
|
| 786 |
+
Step 38470/50462 | Loss: 2.5062 | LR: 6.84e-05 | 1910.4ms/step | 31.4 samples/s
|
| 787 |
+
Step 38480/50462 | Loss: 2.4451 | LR: 6.83e-05 | 1909.4ms/step | 31.4 samples/s
|
| 788 |
+
Step 38490/50462 | Loss: 2.4833 | LR: 6.82e-05 | 1908.7ms/step | 31.4 samples/s
|
| 789 |
+
Step 38500/50462 | Loss: 2.5297 | LR: 6.81e-05 | 1902.5ms/step | 31.5 samples/s
|
| 790 |
+
Validation at step 38500: Loss = 2.5024, Perplexity = 12.21
|
| 791 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38500
|
| 792 |
+
Step 38510/50462 | Loss: 2.4974 | LR: 6.80e-05 | 3239.4ms/step | 18.5 samples/s
|
| 793 |
+
Step 38520/50462 | Loss: 2.4710 | LR: 6.79e-05 | 1900.2ms/step | 31.6 samples/s
|
| 794 |
+
Step 38530/50462 | Loss: 2.4709 | LR: 6.78e-05 | 1905.1ms/step | 31.5 samples/s
|
| 795 |
+
Step 38540/50462 | Loss: 2.4826 | LR: 6.77e-05 | 1907.5ms/step | 31.5 samples/s
|
| 796 |
+
Step 38550/50462 | Loss: 2.4993 | LR: 6.76e-05 | 1910.2ms/step | 31.4 samples/s
|
| 797 |
+
Step 38560/50462 | Loss: 2.4430 | LR: 6.74e-05 | 1910.8ms/step | 31.4 samples/s
|
| 798 |
+
Step 38570/50462 | Loss: 2.4775 | LR: 6.73e-05 | 1912.8ms/step | 31.4 samples/s
|
| 799 |
+
Step 38580/50462 | Loss: 2.4603 | LR: 6.72e-05 | 1902.0ms/step | 31.5 samples/s
|
| 800 |
+
Step 38590/50462 | Loss: 2.5197 | LR: 6.71e-05 | 1903.7ms/step | 31.5 samples/s
|
| 801 |
+
Step 38600/50462 | Loss: 2.4655 | LR: 6.70e-05 | 1903.8ms/step | 31.5 samples/s
|
| 802 |
+
Step 38610/50462 | Loss: 2.4720 | LR: 6.69e-05 | 1906.6ms/step | 31.5 samples/s
|
| 803 |
+
Step 38620/50462 | Loss: 2.4757 | LR: 6.68e-05 | 1906.0ms/step | 31.5 samples/s
|
| 804 |
+
Step 38630/50462 | Loss: 2.4726 | LR: 6.67e-05 | 1904.1ms/step | 31.5 samples/s
|
| 805 |
+
Step 38640/50462 | Loss: 2.5201 | LR: 6.66e-05 | 1903.6ms/step | 31.5 samples/s
|
| 806 |
+
Step 38650/50462 | Loss: 2.5114 | LR: 6.65e-05 | 1904.3ms/step | 31.5 samples/s
|
| 807 |
+
Step 38660/50462 | Loss: 2.4834 | LR: 6.64e-05 | 1908.7ms/step | 31.4 samples/s
|
| 808 |
+
Step 38670/50462 | Loss: 2.4914 | LR: 6.63e-05 | 1908.1ms/step | 31.4 samples/s
|
| 809 |
+
Step 38680/50462 | Loss: 2.4941 | LR: 6.62e-05 | 1910.8ms/step | 31.4 samples/s
|
| 810 |
+
Step 38690/50462 | Loss: 2.5292 | LR: 6.60e-05 | 1915.5ms/step | 31.3 samples/s
|
| 811 |
+
Step 38700/50462 | Loss: 2.4601 | LR: 6.59e-05 | 1912.9ms/step | 31.4 samples/s
|
| 812 |
+
Step 38710/50462 | Loss: 2.4879 | LR: 6.58e-05 | 1912.9ms/step | 31.4 samples/s
|
| 813 |
+
Step 38720/50462 | Loss: 2.4540 | LR: 6.57e-05 | 1907.5ms/step | 31.5 samples/s
|
| 814 |
+
Step 38730/50462 | Loss: 2.5259 | LR: 6.56e-05 | 1910.5ms/step | 31.4 samples/s
|
| 815 |
+
Step 38740/50462 | Loss: 2.4564 | LR: 6.55e-05 | 1910.5ms/step | 31.4 samples/s
|
| 816 |
+
Step 38750/50462 | Loss: 2.5120 | LR: 6.54e-05 | 1909.9ms/step | 31.4 samples/s
|
| 817 |
+
Validation at step 38750: Loss = 2.5020, Perplexity = 12.21
|
| 818 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-38750
|
| 819 |
+
Step 38760/50462 | Loss: 2.4821 | LR: 6.53e-05 | 3208.9ms/step | 18.7 samples/s
|
| 820 |
+
Step 38770/50462 | Loss: 2.4912 | LR: 6.52e-05 | 1897.5ms/step | 31.6 samples/s
|
| 821 |
+
Step 38780/50462 | Loss: 2.4909 | LR: 6.51e-05 | 1903.2ms/step | 31.5 samples/s
|
| 822 |
+
Step 38790/50462 | Loss: 2.4943 | LR: 6.50e-05 | 1906.0ms/step | 31.5 samples/s
|
| 823 |
+
Step 38800/50462 | Loss: 2.4502 | LR: 6.49e-05 | 1905.4ms/step | 31.5 samples/s
|
| 824 |
+
Step 38810/50462 | Loss: 2.4472 | LR: 6.48e-05 | 1907.0ms/step | 31.5 samples/s
|
| 825 |
+
Step 38820/50462 | Loss: 2.5037 | LR: 6.47e-05 | 1911.7ms/step | 31.4 samples/s
|
| 826 |
+
Step 38830/50462 | Loss: 2.5056 | LR: 6.46e-05 | 1909.5ms/step | 31.4 samples/s
|
| 827 |
+
Step 38840/50462 | Loss: 2.4758 | LR: 6.45e-05 | 1909.1ms/step | 31.4 samples/s
|
| 828 |
+
Step 38850/50462 | Loss: 2.4859 | LR: 6.43e-05 | 1914.1ms/step | 31.3 samples/s
|
| 829 |
+
Step 38860/50462 | Loss: 2.4829 | LR: 6.42e-05 | 1915.3ms/step | 31.3 samples/s
|
| 830 |
+
Step 38870/50462 | Loss: 2.4387 | LR: 6.41e-05 | 1911.7ms/step | 31.4 samples/s
|
| 831 |
+
Step 38880/50462 | Loss: 2.4860 | LR: 6.40e-05 | 1903.7ms/step | 31.5 samples/s
|
| 832 |
+
Step 38890/50462 | Loss: 2.4709 | LR: 6.39e-05 | 1909.0ms/step | 31.4 samples/s
|
| 833 |
+
Step 38900/50462 | Loss: 2.4866 | LR: 6.38e-05 | 1914.4ms/step | 31.3 samples/s
|
| 834 |
+
Step 38910/50462 | Loss: 2.4845 | LR: 6.37e-05 | 1913.0ms/step | 31.4 samples/s
|
| 835 |
+
Step 38920/50462 | Loss: 2.5273 | LR: 6.36e-05 | 1913.2ms/step | 31.4 samples/s
|
| 836 |
+
Step 38930/50462 | Loss: 2.4832 | LR: 6.35e-05 | 1911.4ms/step | 31.4 samples/s
|
| 837 |
+
Step 38940/50462 | Loss: 2.4879 | LR: 6.34e-05 | 1914.3ms/step | 31.3 samples/s
|
| 838 |
+
Step 38950/50462 | Loss: 2.5084 | LR: 6.33e-05 | 1914.7ms/step | 31.3 samples/s
|
| 839 |
+
Step 38960/50462 | Loss: 2.4591 | LR: 6.32e-05 | 1911.5ms/step | 31.4 samples/s
|
| 840 |
+
Step 38970/50462 | Loss: 2.4940 | LR: 6.31e-05 | 1913.6ms/step | 31.4 samples/s
|
| 841 |
+
Step 38980/50462 | Loss: 2.4690 | LR: 6.30e-05 | 1911.8ms/step | 31.4 samples/s
|
| 842 |
+
Step 38990/50462 | Loss: 2.4968 | LR: 6.29e-05 | 1911.7ms/step | 31.4 samples/s
|
| 843 |
+
Step 39000/50462 | Loss: 2.4612 | LR: 6.28e-05 | 1913.6ms/step | 31.4 samples/s
|
| 844 |
+
Validation at step 39000: Loss = 2.5016, Perplexity = 12.20
|
| 845 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39000
|
| 846 |
+
Step 39010/50462 | Loss: 2.4515 | LR: 6.27e-05 | 3224.1ms/step | 18.6 samples/s
|
| 847 |
+
Step 39020/50462 | Loss: 2.5099 | LR: 6.26e-05 | 1897.6ms/step | 31.6 samples/s
|
| 848 |
+
Step 39030/50462 | Loss: 2.5208 | LR: 6.25e-05 | 1904.8ms/step | 31.5 samples/s
|
| 849 |
+
Step 39040/50462 | Loss: 2.4325 | LR: 6.23e-05 | 1906.8ms/step | 31.5 samples/s
|
| 850 |
+
Step 39050/50462 | Loss: 2.5027 | LR: 6.22e-05 | 1905.7ms/step | 31.5 samples/s
|
| 851 |
+
Step 39060/50462 | Loss: 2.4958 | LR: 6.21e-05 | 1906.1ms/step | 31.5 samples/s
|
| 852 |
+
Step 39070/50462 | Loss: 2.4695 | LR: 6.20e-05 | 1905.2ms/step | 31.5 samples/s
|
| 853 |
+
Step 39080/50462 | Loss: 2.4896 | LR: 6.19e-05 | 1907.9ms/step | 31.4 samples/s
|
| 854 |
+
Step 39090/50462 | Loss: 2.4618 | LR: 6.18e-05 | 1910.6ms/step | 31.4 samples/s
|
| 855 |
+
Step 39100/50462 | Loss: 2.5075 | LR: 6.17e-05 | 1911.6ms/step | 31.4 samples/s
|
| 856 |
+
Step 39110/50462 | Loss: 2.5073 | LR: 6.16e-05 | 1910.8ms/step | 31.4 samples/s
|
| 857 |
+
Step 39120/50462 | Loss: 2.4912 | LR: 6.15e-05 | 1909.3ms/step | 31.4 samples/s
|
| 858 |
+
Step 39130/50462 | Loss: 2.5231 | LR: 6.14e-05 | 1903.4ms/step | 31.5 samples/s
|
| 859 |
+
Step 39140/50462 | Loss: 2.4319 | LR: 6.13e-05 | 1908.2ms/step | 31.4 samples/s
|
| 860 |
+
Step 39150/50462 | Loss: 2.4830 | LR: 6.12e-05 | 1911.8ms/step | 31.4 samples/s
|
| 861 |
+
Step 39160/50462 | Loss: 2.4999 | LR: 6.11e-05 | 1908.3ms/step | 31.4 samples/s
|
| 862 |
+
Step 39170/50462 | Loss: 2.5056 | LR: 6.10e-05 | 1911.7ms/step | 31.4 samples/s
|
| 863 |
+
Step 39180/50462 | Loss: 2.4659 | LR: 6.09e-05 | 1913.9ms/step | 31.4 samples/s
|
| 864 |
+
Step 39190/50462 | Loss: 2.4421 | LR: 6.08e-05 | 1912.4ms/step | 31.4 samples/s
|
| 865 |
+
Step 39200/50462 | Loss: 2.4794 | LR: 6.07e-05 | 1909.2ms/step | 31.4 samples/s
|
| 866 |
+
Step 39210/50462 | Loss: 2.4959 | LR: 6.06e-05 | 1910.5ms/step | 31.4 samples/s
|
| 867 |
+
Step 39220/50462 | Loss: 2.5357 | LR: 6.05e-05 | 1905.9ms/step | 31.5 samples/s
|
| 868 |
+
Step 39230/50462 | Loss: 2.4372 | LR: 6.04e-05 | 1905.2ms/step | 31.5 samples/s
|
| 869 |
+
Step 39240/50462 | Loss: 2.4938 | LR: 6.03e-05 | 1908.7ms/step | 31.4 samples/s
|
| 870 |
+
Step 39250/50462 | Loss: 2.5061 | LR: 6.02e-05 | 1910.5ms/step | 31.4 samples/s
|
| 871 |
+
Validation at step 39250: Loss = 2.5014, Perplexity = 12.20
|
| 872 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39250
|
| 873 |
+
Step 39260/50462 | Loss: 2.4538 | LR: 6.01e-05 | 3247.1ms/step | 18.5 samples/s
|
| 874 |
+
Step 39270/50462 | Loss: 2.4858 | LR: 6.00e-05 | 1896.4ms/step | 31.6 samples/s
|
| 875 |
+
Step 39280/50462 | Loss: 2.4720 | LR: 5.99e-05 | 1902.2ms/step | 31.5 samples/s
|
| 876 |
+
Step 39290/50462 | Loss: 2.4783 | LR: 5.98e-05 | 1907.9ms/step | 31.4 samples/s
|
| 877 |
+
Step 39300/50462 | Loss: 2.5019 | LR: 5.97e-05 | 1911.0ms/step | 31.4 samples/s
|
| 878 |
+
Step 39310/50462 | Loss: 2.4715 | LR: 5.96e-05 | 1909.9ms/step | 31.4 samples/s
|
| 879 |
+
Step 39320/50462 | Loss: 2.4412 | LR: 5.95e-05 | 1911.9ms/step | 31.4 samples/s
|
| 880 |
+
Step 39330/50462 | Loss: 2.4685 | LR: 5.94e-05 | 1911.5ms/step | 31.4 samples/s
|
| 881 |
+
Step 39340/50462 | Loss: 2.4567 | LR: 5.93e-05 | 1914.1ms/step | 31.3 samples/s
|
| 882 |
+
Step 39350/50462 | Loss: 2.4819 | LR: 5.91e-05 | 1914.9ms/step | 31.3 samples/s
|
| 883 |
+
Step 39360/50462 | Loss: 2.4862 | LR: 5.90e-05 | 1912.3ms/step | 31.4 samples/s
|
| 884 |
+
Step 39370/50462 | Loss: 2.4971 | LR: 5.89e-05 | 1910.4ms/step | 31.4 samples/s
|
| 885 |
+
Step 39380/50462 | Loss: 2.5012 | LR: 5.88e-05 | 1909.6ms/step | 31.4 samples/s
|
| 886 |
+
Step 39390/50462 | Loss: 2.4657 | LR: 5.87e-05 | 1910.5ms/step | 31.4 samples/s
|
| 887 |
+
Step 39400/50462 | Loss: 2.4930 | LR: 5.86e-05 | 1914.3ms/step | 31.3 samples/s
|
| 888 |
+
Step 39410/50462 | Loss: 2.4845 | LR: 5.85e-05 | 1911.7ms/step | 31.4 samples/s
|
| 889 |
+
Step 39420/50462 | Loss: 2.4328 | LR: 5.84e-05 | 1915.1ms/step | 31.3 samples/s
|
| 890 |
+
Step 39430/50462 | Loss: 2.4467 | LR: 5.83e-05 | 1915.7ms/step | 31.3 samples/s
|
| 891 |
+
Step 39440/50462 | Loss: 2.4943 | LR: 5.82e-05 | 1914.8ms/step | 31.3 samples/s
|
| 892 |
+
Step 39450/50462 | Loss: 2.4871 | LR: 5.81e-05 | 1911.5ms/step | 31.4 samples/s
|
| 893 |
+
Step 39460/50462 | Loss: 2.4951 | LR: 5.80e-05 | 1912.2ms/step | 31.4 samples/s
|
| 894 |
+
Step 39470/50462 | Loss: 2.5121 | LR: 5.79e-05 | 1912.2ms/step | 31.4 samples/s
|
| 895 |
+
Step 39480/50462 | Loss: 2.4723 | LR: 5.78e-05 | 1915.9ms/step | 31.3 samples/s
|
| 896 |
+
Step 39490/50462 | Loss: 2.5062 | LR: 5.77e-05 | 1913.2ms/step | 31.4 samples/s
|
| 897 |
+
Step 39500/50462 | Loss: 2.4814 | LR: 5.76e-05 | 1910.8ms/step | 31.4 samples/s
|
| 898 |
+
Validation at step 39500: Loss = 2.5012, Perplexity = 12.20
|
| 899 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39500
|
| 900 |
+
Step 39510/50462 | Loss: 2.4763 | LR: 5.75e-05 | 3263.4ms/step | 18.4 samples/s
|
| 901 |
+
Step 39520/50462 | Loss: 2.4645 | LR: 5.74e-05 | 1903.5ms/step | 31.5 samples/s
|
| 902 |
+
Step 39530/50462 | Loss: 2.4750 | LR: 5.73e-05 | 1908.4ms/step | 31.4 samples/s
|
| 903 |
+
Step 39540/50462 | Loss: 2.4553 | LR: 5.72e-05 | 1911.8ms/step | 31.4 samples/s
|
| 904 |
+
Step 39550/50462 | Loss: 2.4819 | LR: 5.71e-05 | 1906.4ms/step | 31.5 samples/s
|
| 905 |
+
Step 39560/50462 | Loss: 2.5036 | LR: 5.70e-05 | 1907.8ms/step | 31.4 samples/s
|
| 906 |
+
Step 39570/50462 | Loss: 2.4583 | LR: 5.69e-05 | 1915.9ms/step | 31.3 samples/s
|
| 907 |
+
Step 39580/50462 | Loss: 2.4344 | LR: 5.68e-05 | 1915.2ms/step | 31.3 samples/s
|
| 908 |
+
Step 39590/50462 | Loss: 2.4984 | LR: 5.67e-05 | 1913.5ms/step | 31.4 samples/s
|
| 909 |
+
Step 39600/50462 | Loss: 2.4601 | LR: 5.66e-05 | 1914.9ms/step | 31.3 samples/s
|
| 910 |
+
Step 39610/50462 | Loss: 2.4766 | LR: 5.65e-05 | 1910.0ms/step | 31.4 samples/s
|
| 911 |
+
Step 39620/50462 | Loss: 2.4694 | LR: 5.64e-05 | 1914.9ms/step | 31.3 samples/s
|
| 912 |
+
Step 39630/50462 | Loss: 2.4847 | LR: 5.63e-05 | 1916.7ms/step | 31.3 samples/s
|
| 913 |
+
Step 39640/50462 | Loss: 2.5145 | LR: 5.62e-05 | 1920.6ms/step | 31.2 samples/s
|
| 914 |
+
Step 39650/50462 | Loss: 2.4727 | LR: 5.61e-05 | 1917.1ms/step | 31.3 samples/s
|
| 915 |
+
Step 39660/50462 | Loss: 2.4678 | LR: 5.60e-05 | 1910.2ms/step | 31.4 samples/s
|
| 916 |
+
Step 39670/50462 | Loss: 2.4771 | LR: 5.59e-05 | 1906.5ms/step | 31.5 samples/s
|
| 917 |
+
Step 39680/50462 | Loss: 2.5100 | LR: 5.58e-05 | 1911.1ms/step | 31.4 samples/s
|
| 918 |
+
Step 39690/50462 | Loss: 2.5413 | LR: 5.57e-05 | 1910.9ms/step | 31.4 samples/s
|
| 919 |
+
Step 39700/50462 | Loss: 2.5118 | LR: 5.56e-05 | 1910.7ms/step | 31.4 samples/s
|
| 920 |
+
Step 39710/50462 | Loss: 2.5255 | LR: 5.55e-05 | 1908.0ms/step | 31.4 samples/s
|
| 921 |
+
Step 39720/50462 | Loss: 2.4972 | LR: 5.54e-05 | 1906.4ms/step | 31.5 samples/s
|
| 922 |
+
Step 39730/50462 | Loss: 2.5244 | LR: 5.53e-05 | 1910.0ms/step | 31.4 samples/s
|
| 923 |
+
Step 39740/50462 | Loss: 2.4707 | LR: 5.52e-05 | 1908.0ms/step | 31.4 samples/s
|
| 924 |
+
Step 39750/50462 | Loss: 2.4595 | LR: 5.51e-05 | 1912.8ms/step | 31.4 samples/s
|
| 925 |
+
Validation at step 39750: Loss = 2.5009, Perplexity = 12.19
|
| 926 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-39750
|
| 927 |
+
Step 39760/50462 | Loss: 2.4977 | LR: 5.50e-05 | 3226.8ms/step | 18.6 samples/s
|
| 928 |
+
Step 39770/50462 | Loss: 2.4665 | LR: 5.49e-05 | 1901.4ms/step | 31.6 samples/s
|
| 929 |
+
Step 39780/50462 | Loss: 2.4693 | LR: 5.48e-05 | 1903.5ms/step | 31.5 samples/s
|
| 930 |
+
Step 39790/50462 | Loss: 2.4519 | LR: 5.47e-05 | 1903.6ms/step | 31.5 samples/s
|
| 931 |
+
Step 39800/50462 | Loss: 2.4929 | LR: 5.46e-05 | 1909.2ms/step | 31.4 samples/s
|
| 932 |
+
Step 39810/50462 | Loss: 2.5056 | LR: 5.45e-05 | 1913.6ms/step | 31.4 samples/s
|
| 933 |
+
Step 39820/50462 | Loss: 2.4870 | LR: 5.44e-05 | 1916.2ms/step | 31.3 samples/s
|
| 934 |
+
Step 39830/50462 | Loss: 2.4924 | LR: 5.43e-05 | 1913.8ms/step | 31.4 samples/s
|
| 935 |
+
Step 39840/50462 | Loss: 2.5085 | LR: 5.42e-05 | 1915.0ms/step | 31.3 samples/s
|
| 936 |
+
Step 39850/50462 | Loss: 2.5134 | LR: 5.41e-05 | 1904.3ms/step | 31.5 samples/s
|
| 937 |
+
Step 39860/50462 | Loss: 2.5015 | LR: 5.40e-05 | 1906.5ms/step | 31.5 samples/s
|
| 938 |
+
Step 39870/50462 | Loss: 2.4831 | LR: 5.39e-05 | 1906.3ms/step | 31.5 samples/s
|
| 939 |
+
Step 39880/50462 | Loss: 2.4957 | LR: 5.38e-05 | 1911.0ms/step | 31.4 samples/s
|
| 940 |
+
Step 39890/50462 | Loss: 2.4954 | LR: 5.38e-05 | 1913.3ms/step | 31.4 samples/s
|
| 941 |
+
Step 39900/50462 | Loss: 2.5022 | LR: 5.37e-05 | 1916.7ms/step | 31.3 samples/s
|
| 942 |
+
Step 39910/50462 | Loss: 2.4859 | LR: 5.36e-05 | 1915.3ms/step | 31.3 samples/s
|
| 943 |
+
Step 39920/50462 | Loss: 2.4717 | LR: 5.35e-05 | 1915.2ms/step | 31.3 samples/s
|
| 944 |
+
Step 39930/50462 | Loss: 2.4417 | LR: 5.34e-05 | 1918.0ms/step | 31.3 samples/s
|
| 945 |
+
Step 39940/50462 | Loss: 2.4867 | LR: 5.33e-05 | 1918.0ms/step | 31.3 samples/s
|
| 946 |
+
Step 39950/50462 | Loss: 2.4999 | LR: 5.32e-05 | 1917.2ms/step | 31.3 samples/s
|
| 947 |
+
Step 39960/50462 | Loss: 2.4942 | LR: 5.31e-05 | 1902.0ms/step | 31.5 samples/s
|
| 948 |
+
Step 39970/50462 | Loss: 2.4774 | LR: 5.30e-05 | 1904.0ms/step | 31.5 samples/s
|
| 949 |
+
Step 39980/50462 | Loss: 2.5024 | LR: 5.29e-05 | 1902.8ms/step | 31.5 samples/s
|
| 950 |
+
Step 39990/50462 | Loss: 2.4945 | LR: 5.28e-05 | 1905.3ms/step | 31.5 samples/s
|
| 951 |
+
Step 40000/50462 | Loss: 2.5008 | LR: 5.27e-05 | 1911.7ms/step | 31.4 samples/s
|
| 952 |
+
Validation at step 40000: Loss = 2.5006, Perplexity = 12.19
|
| 953 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40000
|
| 954 |
+
Step 40010/50462 | Loss: 2.4687 | LR: 5.26e-05 | 3260.7ms/step | 18.4 samples/s
|
| 955 |
+
Step 40020/50462 | Loss: 2.5041 | LR: 5.25e-05 | 1899.1ms/step | 31.6 samples/s
|
| 956 |
+
Step 40030/50462 | Loss: 2.4762 | LR: 5.24e-05 | 1903.6ms/step | 31.5 samples/s
|
| 957 |
+
Step 40040/50462 | Loss: 2.4697 | LR: 5.23e-05 | 1902.0ms/step | 31.5 samples/s
|
| 958 |
+
Step 40050/50462 | Loss: 2.4677 | LR: 5.22e-05 | 1905.3ms/step | 31.5 samples/s
|
| 959 |
+
Step 40060/50462 | Loss: 2.5143 | LR: 5.21e-05 | 1912.8ms/step | 31.4 samples/s
|
| 960 |
+
Step 40070/50462 | Loss: 2.5047 | LR: 5.20e-05 | 1914.7ms/step | 31.3 samples/s
|
| 961 |
+
Step 40080/50462 | Loss: 2.4890 | LR: 5.19e-05 | 1913.1ms/step | 31.4 samples/s
|
| 962 |
+
Step 40090/50462 | Loss: 2.4571 | LR: 5.18e-05 | 1914.0ms/step | 31.3 samples/s
|
| 963 |
+
Step 40100/50462 | Loss: 2.4327 | LR: 5.17e-05 | 1911.6ms/step | 31.4 samples/s
|
| 964 |
+
Step 40110/50462 | Loss: 2.5371 | LR: 5.16e-05 | 1905.9ms/step | 31.5 samples/s
|
| 965 |
+
Step 40120/50462 | Loss: 2.5007 | LR: 5.15e-05 | 1901.7ms/step | 31.6 samples/s
|
| 966 |
+
Step 40130/50462 | Loss: 2.4701 | LR: 5.14e-05 | 1905.1ms/step | 31.5 samples/s
|
| 967 |
+
Step 40140/50462 | Loss: 2.4891 | LR: 5.13e-05 | 1908.7ms/step | 31.4 samples/s
|
| 968 |
+
Step 40150/50462 | Loss: 2.5014 | LR: 5.12e-05 | 1911.7ms/step | 31.4 samples/s
|
| 969 |
+
Step 40160/50462 | Loss: 2.4826 | LR: 5.11e-05 | 1911.2ms/step | 31.4 samples/s
|
| 970 |
+
Step 40170/50462 | Loss: 2.4291 | LR: 5.10e-05 | 1913.5ms/step | 31.4 samples/s
|
| 971 |
+
Step 40180/50462 | Loss: 2.4815 | LR: 5.09e-05 | 1915.0ms/step | 31.3 samples/s
|
| 972 |
+
Step 40190/50462 | Loss: 2.5165 | LR: 5.09e-05 | 1912.2ms/step | 31.4 samples/s
|
| 973 |
+
Step 40200/50462 | Loss: 2.4453 | LR: 5.08e-05 | 1908.8ms/step | 31.4 samples/s
|
| 974 |
+
Step 40210/50462 | Loss: 2.4965 | LR: 5.07e-05 | 1909.1ms/step | 31.4 samples/s
|
| 975 |
+
Step 40220/50462 | Loss: 2.5077 | LR: 5.06e-05 | 1911.0ms/step | 31.4 samples/s
|
| 976 |
+
Step 40230/50462 | Loss: 2.4971 | LR: 5.05e-05 | 1913.1ms/step | 31.4 samples/s
|
| 977 |
+
Step 40240/50462 | Loss: 2.4656 | LR: 5.04e-05 | 1918.3ms/step | 31.3 samples/s
|
| 978 |
+
Step 40250/50462 | Loss: 2.5047 | LR: 5.03e-05 | 1914.9ms/step | 31.3 samples/s
|
| 979 |
+
Validation at step 40250: Loss = 2.5002, Perplexity = 12.18
|
| 980 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40250
|
| 981 |
+
Step 40260/50462 | Loss: 2.4563 | LR: 5.02e-05 | 3240.6ms/step | 18.5 samples/s
|
| 982 |
+
Step 40270/50462 | Loss: 2.5183 | LR: 5.01e-05 | 1901.5ms/step | 31.6 samples/s
|
| 983 |
+
Step 40280/50462 | Loss: 2.4729 | LR: 5.00e-05 | 1905.1ms/step | 31.5 samples/s
|
| 984 |
+
Step 40290/50462 | Loss: 2.4348 | LR: 4.99e-05 | 1910.7ms/step | 31.4 samples/s
|
| 985 |
+
Step 40300/50462 | Loss: 2.4898 | LR: 4.98e-05 | 1911.3ms/step | 31.4 samples/s
|
| 986 |
+
Step 40310/50462 | Loss: 2.4858 | LR: 4.97e-05 | 1913.7ms/step | 31.4 samples/s
|
| 987 |
+
Step 40320/50462 | Loss: 2.5080 | LR: 4.96e-05 | 1912.1ms/step | 31.4 samples/s
|
| 988 |
+
Step 40330/50462 | Loss: 2.4797 | LR: 4.95e-05 | 1916.3ms/step | 31.3 samples/s
|
| 989 |
+
Step 40340/50462 | Loss: 2.4769 | LR: 4.94e-05 | 1916.1ms/step | 31.3 samples/s
|
| 990 |
+
Step 40350/50462 | Loss: 2.4420 | LR: 4.93e-05 | 1917.7ms/step | 31.3 samples/s
|
| 991 |
+
Step 40360/50462 | Loss: 2.5301 | LR: 4.92e-05 | 1912.7ms/step | 31.4 samples/s
|
| 992 |
+
Step 40370/50462 | Loss: 2.4907 | LR: 4.91e-05 | 1917.3ms/step | 31.3 samples/s
|
| 993 |
+
Step 40380/50462 | Loss: 2.4765 | LR: 4.90e-05 | 1916.6ms/step | 31.3 samples/s
|
| 994 |
+
Step 40390/50462 | Loss: 2.4579 | LR: 4.90e-05 | 1914.0ms/step | 31.3 samples/s
|
| 995 |
+
Step 40400/50462 | Loss: 2.4924 | LR: 4.89e-05 | 1909.0ms/step | 31.4 samples/s
|
| 996 |
+
Step 40410/50462 | Loss: 2.5231 | LR: 4.88e-05 | 1904.6ms/step | 31.5 samples/s
|
| 997 |
+
Step 40420/50462 | Loss: 2.4860 | LR: 4.87e-05 | 1903.1ms/step | 31.5 samples/s
|
| 998 |
+
Step 40430/50462 | Loss: 2.4883 | LR: 4.86e-05 | 1905.1ms/step | 31.5 samples/s
|
| 999 |
+
Step 40440/50462 | Loss: 2.4776 | LR: 4.85e-05 | 1912.6ms/step | 31.4 samples/s
|
| 1000 |
+
Step 40450/50462 | Loss: 2.4760 | LR: 4.84e-05 | 1916.2ms/step | 31.3 samples/s
|
| 1001 |
+
Step 40460/50462 | Loss: 2.4932 | LR: 4.83e-05 | 1915.8ms/step | 31.3 samples/s
|
| 1002 |
+
Step 40470/50462 | Loss: 2.4538 | LR: 4.82e-05 | 1901.9ms/step | 31.5 samples/s
|
| 1003 |
+
Step 40480/50462 | Loss: 2.4878 | LR: 4.81e-05 | 1906.4ms/step | 31.5 samples/s
|
| 1004 |
+
Step 40490/50462 | Loss: 2.4864 | LR: 4.80e-05 | 1910.0ms/step | 31.4 samples/s
|
| 1005 |
+
Step 40500/50462 | Loss: 2.4189 | LR: 4.79e-05 | 1912.9ms/step | 31.4 samples/s
|
| 1006 |
+
Validation at step 40500: Loss = 2.5000, Perplexity = 12.18
|
| 1007 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40500
|
| 1008 |
+
Step 40510/50462 | Loss: 2.5056 | LR: 4.78e-05 | 3230.3ms/step | 18.6 samples/s
|
| 1009 |
+
Step 40520/50462 | Loss: 2.4979 | LR: 4.77e-05 | 1898.3ms/step | 31.6 samples/s
|
| 1010 |
+
Step 40530/50462 | Loss: 2.4717 | LR: 4.76e-05 | 1906.2ms/step | 31.5 samples/s
|
| 1011 |
+
Step 40540/50462 | Loss: 2.4439 | LR: 4.76e-05 | 1905.6ms/step | 31.5 samples/s
|
| 1012 |
+
Step 40550/50462 | Loss: 2.4692 | LR: 4.75e-05 | 1908.4ms/step | 31.4 samples/s
|
| 1013 |
+
Step 40560/50462 | Loss: 2.4978 | LR: 4.74e-05 | 1910.7ms/step | 31.4 samples/s
|
| 1014 |
+
Step 40570/50462 | Loss: 2.5075 | LR: 4.73e-05 | 1914.0ms/step | 31.3 samples/s
|
| 1015 |
+
Step 40580/50462 | Loss: 2.4818 | LR: 4.72e-05 | 1905.4ms/step | 31.5 samples/s
|
| 1016 |
+
Step 40590/50462 | Loss: 2.4869 | LR: 4.71e-05 | 1911.7ms/step | 31.4 samples/s
|
| 1017 |
+
Step 40600/50462 | Loss: 2.4530 | LR: 4.70e-05 | 1911.7ms/step | 31.4 samples/s
|
| 1018 |
+
Step 40610/50462 | Loss: 2.4656 | LR: 4.69e-05 | 1913.0ms/step | 31.4 samples/s
|
| 1019 |
+
Step 40620/50462 | Loss: 2.5198 | LR: 4.68e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1020 |
+
Step 40630/50462 | Loss: 2.4830 | LR: 4.67e-05 | 1906.9ms/step | 31.5 samples/s
|
| 1021 |
+
Step 40640/50462 | Loss: 2.4674 | LR: 4.66e-05 | 1911.3ms/step | 31.4 samples/s
|
| 1022 |
+
Step 40650/50462 | Loss: 2.4968 | LR: 4.65e-05 | 1906.6ms/step | 31.5 samples/s
|
| 1023 |
+
Step 40660/50462 | Loss: 2.5126 | LR: 4.64e-05 | 1907.4ms/step | 31.5 samples/s
|
| 1024 |
+
Step 40670/50462 | Loss: 2.4803 | LR: 4.64e-05 | 1908.0ms/step | 31.4 samples/s
|
| 1025 |
+
Step 40680/50462 | Loss: 2.4766 | LR: 4.63e-05 | 1909.2ms/step | 31.4 samples/s
|
| 1026 |
+
Step 40690/50462 | Loss: 2.4978 | LR: 4.62e-05 | 1912.3ms/step | 31.4 samples/s
|
| 1027 |
+
Step 40700/50462 | Loss: 2.5106 | LR: 4.61e-05 | 1911.1ms/step | 31.4 samples/s
|
| 1028 |
+
Step 40710/50462 | Loss: 2.4478 | LR: 4.60e-05 | 1908.9ms/step | 31.4 samples/s
|
| 1029 |
+
Step 40720/50462 | Loss: 2.5281 | LR: 4.59e-05 | 1912.8ms/step | 31.4 samples/s
|
| 1030 |
+
Step 40730/50462 | Loss: 2.4716 | LR: 4.58e-05 | 1913.2ms/step | 31.4 samples/s
|
| 1031 |
+
Step 40740/50462 | Loss: 2.4810 | LR: 4.57e-05 | 1914.0ms/step | 31.3 samples/s
|
| 1032 |
+
Step 40750/50462 | Loss: 2.4979 | LR: 4.56e-05 | 1916.9ms/step | 31.3 samples/s
|
| 1033 |
+
Validation at step 40750: Loss = 2.4997, Perplexity = 12.18
|
| 1034 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-40750
|
| 1035 |
+
Step 40760/50462 | Loss: 2.5134 | LR: 4.55e-05 | 3211.0ms/step | 18.7 samples/s
|
| 1036 |
+
Step 40770/50462 | Loss: 2.4836 | LR: 4.54e-05 | 1901.0ms/step | 31.6 samples/s
|
| 1037 |
+
Step 40780/50462 | Loss: 2.5233 | LR: 4.54e-05 | 1906.4ms/step | 31.5 samples/s
|
| 1038 |
+
Step 40790/50462 | Loss: 2.4604 | LR: 4.53e-05 | 1912.5ms/step | 31.4 samples/s
|
| 1039 |
+
Step 40800/50462 | Loss: 2.4747 | LR: 4.52e-05 | 1914.2ms/step | 31.3 samples/s
|
| 1040 |
+
Step 40810/50462 | Loss: 2.4702 | LR: 4.51e-05 | 1915.8ms/step | 31.3 samples/s
|
| 1041 |
+
Step 40820/50462 | Loss: 2.4585 | LR: 4.50e-05 | 1912.4ms/step | 31.4 samples/s
|
| 1042 |
+
Step 40830/50462 | Loss: 2.4644 | LR: 4.49e-05 | 1912.6ms/step | 31.4 samples/s
|
| 1043 |
+
Step 40840/50462 | Loss: 2.4614 | LR: 4.48e-05 | 1902.5ms/step | 31.5 samples/s
|
| 1044 |
+
Step 40850/50462 | Loss: 2.4527 | LR: 4.47e-05 | 1906.5ms/step | 31.5 samples/s
|
| 1045 |
+
Step 40860/50462 | Loss: 2.5000 | LR: 4.46e-05 | 1908.6ms/step | 31.4 samples/s
|
| 1046 |
+
Step 40870/50462 | Loss: 2.4343 | LR: 4.45e-05 | 1912.9ms/step | 31.4 samples/s
|
| 1047 |
+
Step 40880/50462 | Loss: 2.4310 | LR: 4.45e-05 | 1912.9ms/step | 31.4 samples/s
|
| 1048 |
+
Step 40890/50462 | Loss: 2.4309 | LR: 4.44e-05 | 1906.6ms/step | 31.5 samples/s
|
| 1049 |
+
Step 40900/50462 | Loss: 2.5144 | LR: 4.43e-05 | 1908.3ms/step | 31.4 samples/s
|
| 1050 |
+
Step 40910/50462 | Loss: 2.4829 | LR: 4.42e-05 | 1910.8ms/step | 31.4 samples/s
|
| 1051 |
+
Step 40920/50462 | Loss: 2.4753 | LR: 4.41e-05 | 1912.8ms/step | 31.4 samples/s
|
| 1052 |
+
Step 40930/50462 | Loss: 2.4875 | LR: 4.40e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1053 |
+
Step 40940/50462 | Loss: 2.4693 | LR: 4.39e-05 | 1904.5ms/step | 31.5 samples/s
|
| 1054 |
+
Step 40950/50462 | Loss: 2.4746 | LR: 4.38e-05 | 1909.0ms/step | 31.4 samples/s
|
| 1055 |
+
Step 40960/50462 | Loss: 2.4727 | LR: 4.37e-05 | 1909.7ms/step | 31.4 samples/s
|
| 1056 |
+
Step 40970/50462 | Loss: 2.4505 | LR: 4.36e-05 | 1913.4ms/step | 31.4 samples/s
|
| 1057 |
+
Step 40980/50462 | Loss: 2.4781 | LR: 4.36e-05 | 1914.6ms/step | 31.3 samples/s
|
| 1058 |
+
Step 40990/50462 | Loss: 2.5150 | LR: 4.35e-05 | 1913.9ms/step | 31.3 samples/s
|
| 1059 |
+
Step 41000/50462 | Loss: 2.4566 | LR: 4.34e-05 | 1914.1ms/step | 31.3 samples/s
|
| 1060 |
+
Validation at step 41000: Loss = 2.4996, Perplexity = 12.18
|
| 1061 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41000
|
| 1062 |
+
Step 41010/50462 | Loss: 2.4533 | LR: 4.33e-05 | 3235.5ms/step | 18.5 samples/s
|
| 1063 |
+
Step 41020/50462 | Loss: 2.4536 | LR: 4.32e-05 | 1898.3ms/step | 31.6 samples/s
|
| 1064 |
+
Step 41030/50462 | Loss: 2.4607 | LR: 4.31e-05 | 1901.1ms/step | 31.6 samples/s
|
| 1065 |
+
Step 41040/50462 | Loss: 2.4620 | LR: 4.30e-05 | 1907.3ms/step | 31.5 samples/s
|
| 1066 |
+
Step 41050/50462 | Loss: 2.4606 | LR: 4.29e-05 | 1906.4ms/step | 31.5 samples/s
|
| 1067 |
+
Step 41060/50462 | Loss: 2.4950 | LR: 4.28e-05 | 1908.7ms/step | 31.4 samples/s
|
| 1068 |
+
Step 41070/50462 | Loss: 2.5082 | LR: 4.28e-05 | 1908.5ms/step | 31.4 samples/s
|
| 1069 |
+
Step 41080/50462 | Loss: 2.5029 | LR: 4.27e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1070 |
+
Step 41090/50462 | Loss: 2.5107 | LR: 4.26e-05 | 1910.3ms/step | 31.4 samples/s
|
| 1071 |
+
Step 41100/50462 | Loss: 2.4698 | LR: 4.25e-05 | 1910.4ms/step | 31.4 samples/s
|
| 1072 |
+
Step 41110/50462 | Loss: 2.5057 | LR: 4.24e-05 | 1909.2ms/step | 31.4 samples/s
|
| 1073 |
+
Step 41120/50462 | Loss: 2.5104 | LR: 4.23e-05 | 1908.2ms/step | 31.4 samples/s
|
| 1074 |
+
Step 41130/50462 | Loss: 2.4433 | LR: 4.22e-05 | 1901.5ms/step | 31.6 samples/s
|
| 1075 |
+
Step 41140/50462 | Loss: 2.5001 | LR: 4.21e-05 | 1903.7ms/step | 31.5 samples/s
|
| 1076 |
+
Step 41150/50462 | Loss: 2.4943 | LR: 4.21e-05 | 1908.0ms/step | 31.4 samples/s
|
| 1077 |
+
Step 41160/50462 | Loss: 2.4574 | LR: 4.20e-05 | 1910.6ms/step | 31.4 samples/s
|
| 1078 |
+
Step 41170/50462 | Loss: 2.4784 | LR: 4.19e-05 | 1905.9ms/step | 31.5 samples/s
|
| 1079 |
+
Step 41180/50462 | Loss: 2.5213 | LR: 4.18e-05 | 1904.8ms/step | 31.5 samples/s
|
| 1080 |
+
Step 41190/50462 | Loss: 2.5099 | LR: 4.17e-05 | 1906.8ms/step | 31.5 samples/s
|
| 1081 |
+
Step 41200/50462 | Loss: 2.5087 | LR: 4.16e-05 | 1905.5ms/step | 31.5 samples/s
|
| 1082 |
+
Step 41210/50462 | Loss: 2.4915 | LR: 4.15e-05 | 1912.3ms/step | 31.4 samples/s
|
| 1083 |
+
Step 41220/50462 | Loss: 2.4834 | LR: 4.14e-05 | 1909.6ms/step | 31.4 samples/s
|
| 1084 |
+
Step 41230/50462 | Loss: 2.4764 | LR: 4.14e-05 | 1905.9ms/step | 31.5 samples/s
|
| 1085 |
+
Step 41240/50462 | Loss: 2.4966 | LR: 4.13e-05 | 1905.8ms/step | 31.5 samples/s
|
| 1086 |
+
Step 41250/50462 | Loss: 2.5131 | LR: 4.12e-05 | 1908.2ms/step | 31.4 samples/s
|
| 1087 |
+
Validation at step 41250: Loss = 2.4995, Perplexity = 12.18
|
| 1088 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41250
|
| 1089 |
+
Step 41260/50462 | Loss: 2.4909 | LR: 4.11e-05 | 3249.2ms/step | 18.5 samples/s
|
| 1090 |
+
Step 41270/50462 | Loss: 2.4871 | LR: 4.10e-05 | 1899.0ms/step | 31.6 samples/s
|
| 1091 |
+
Step 41280/50462 | Loss: 2.4565 | LR: 4.09e-05 | 1905.2ms/step | 31.5 samples/s
|
| 1092 |
+
Step 41290/50462 | Loss: 2.4331 | LR: 4.08e-05 | 1907.6ms/step | 31.5 samples/s
|
| 1093 |
+
Step 41300/50462 | Loss: 2.4720 | LR: 4.07e-05 | 1911.3ms/step | 31.4 samples/s
|
| 1094 |
+
Step 41310/50462 | Loss: 2.4622 | LR: 4.07e-05 | 1910.8ms/step | 31.4 samples/s
|
| 1095 |
+
Step 41320/50462 | Loss: 2.4223 | LR: 4.06e-05 | 1910.5ms/step | 31.4 samples/s
|
| 1096 |
+
Step 41330/50462 | Loss: 2.4394 | LR: 4.05e-05 | 1912.8ms/step | 31.4 samples/s
|
| 1097 |
+
Step 41340/50462 | Loss: 2.4939 | LR: 4.04e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1098 |
+
Step 41350/50462 | Loss: 2.4919 | LR: 4.03e-05 | 1904.7ms/step | 31.5 samples/s
|
| 1099 |
+
Step 41360/50462 | Loss: 2.4293 | LR: 4.02e-05 | 1909.0ms/step | 31.4 samples/s
|
| 1100 |
+
Step 41370/50462 | Loss: 2.4568 | LR: 4.01e-05 | 1913.9ms/step | 31.3 samples/s
|
| 1101 |
+
Step 41380/50462 | Loss: 2.4334 | LR: 4.01e-05 | 1908.8ms/step | 31.4 samples/s
|
| 1102 |
+
Step 41390/50462 | Loss: 2.4303 | LR: 4.00e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1103 |
+
Step 41400/50462 | Loss: 2.4492 | LR: 3.99e-05 | 1916.7ms/step | 31.3 samples/s
|
| 1104 |
+
Step 41410/50462 | Loss: 2.4324 | LR: 3.98e-05 | 1909.5ms/step | 31.4 samples/s
|
| 1105 |
+
Step 41420/50462 | Loss: 2.4122 | LR: 3.97e-05 | 1911.5ms/step | 31.4 samples/s
|
| 1106 |
+
Step 41430/50462 | Loss: 2.4613 | LR: 3.96e-05 | 1909.9ms/step | 31.4 samples/s
|
| 1107 |
+
Step 41440/50462 | Loss: 2.4220 | LR: 3.95e-05 | 1911.3ms/step | 31.4 samples/s
|
| 1108 |
+
Step 41450/50462 | Loss: 2.4919 | LR: 3.95e-05 | 1912.9ms/step | 31.4 samples/s
|
| 1109 |
+
Step 41460/50462 | Loss: 2.4326 | LR: 3.94e-05 | 1916.9ms/step | 31.3 samples/s
|
| 1110 |
+
Step 41470/50462 | Loss: 2.4562 | LR: 3.93e-05 | 1920.4ms/step | 31.2 samples/s
|
| 1111 |
+
Step 41480/50462 | Loss: 2.4470 | LR: 3.92e-05 | 1916.4ms/step | 31.3 samples/s
|
| 1112 |
+
Step 41490/50462 | Loss: 2.4286 | LR: 3.91e-05 | 1920.2ms/step | 31.2 samples/s
|
| 1113 |
+
Step 41500/50462 | Loss: 2.5120 | LR: 3.90e-05 | 1911.6ms/step | 31.4 samples/s
|
| 1114 |
+
Validation at step 41500: Loss = 2.4999, Perplexity = 12.18
|
| 1115 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41500
|
| 1116 |
+
Step 41510/50462 | Loss: 2.4855 | LR: 3.90e-05 | 3267.7ms/step | 18.4 samples/s
|
| 1117 |
+
Step 41520/50462 | Loss: 2.4791 | LR: 3.89e-05 | 1900.7ms/step | 31.6 samples/s
|
| 1118 |
+
Step 41530/50462 | Loss: 2.4304 | LR: 3.88e-05 | 1909.6ms/step | 31.4 samples/s
|
| 1119 |
+
Step 41540/50462 | Loss: 2.4501 | LR: 3.87e-05 | 1909.3ms/step | 31.4 samples/s
|
| 1120 |
+
Step 41550/50462 | Loss: 2.4246 | LR: 3.86e-05 | 1907.8ms/step | 31.5 samples/s
|
| 1121 |
+
Step 41560/50462 | Loss: 2.4521 | LR: 3.85e-05 | 1909.5ms/step | 31.4 samples/s
|
| 1122 |
+
Step 41570/50462 | Loss: 2.4463 | LR: 3.84e-05 | 1914.7ms/step | 31.3 samples/s
|
| 1123 |
+
Step 41580/50462 | Loss: 2.4679 | LR: 3.84e-05 | 1916.1ms/step | 31.3 samples/s
|
| 1124 |
+
Step 41590/50462 | Loss: 2.4522 | LR: 3.83e-05 | 1910.7ms/step | 31.4 samples/s
|
| 1125 |
+
Step 41600/50462 | Loss: 2.4365 | LR: 3.82e-05 | 1914.1ms/step | 31.3 samples/s
|
| 1126 |
+
Step 41610/50462 | Loss: 2.4415 | LR: 3.81e-05 | 1912.4ms/step | 31.4 samples/s
|
| 1127 |
+
Step 41620/50462 | Loss: 2.4759 | LR: 3.80e-05 | 1917.6ms/step | 31.3 samples/s
|
| 1128 |
+
Step 41630/50462 | Loss: 2.4692 | LR: 3.79e-05 | 1915.0ms/step | 31.3 samples/s
|
| 1129 |
+
Step 41640/50462 | Loss: 2.4616 | LR: 3.79e-05 | 1909.4ms/step | 31.4 samples/s
|
| 1130 |
+
Step 41650/50462 | Loss: 2.4704 | LR: 3.78e-05 | 1913.8ms/step | 31.4 samples/s
|
| 1131 |
+
Step 41660/50462 | Loss: 2.4468 | LR: 3.77e-05 | 1910.4ms/step | 31.4 samples/s
|
| 1132 |
+
Step 41670/50462 | Loss: 2.4982 | LR: 3.76e-05 | 1915.5ms/step | 31.3 samples/s
|
| 1133 |
+
Step 41680/50462 | Loss: 2.4105 | LR: 3.75e-05 | 1915.6ms/step | 31.3 samples/s
|
| 1134 |
+
Step 41690/50462 | Loss: 2.4330 | LR: 3.74e-05 | 1911.7ms/step | 31.4 samples/s
|
| 1135 |
+
Step 41700/50462 | Loss: 2.4075 | LR: 3.74e-05 | 1908.8ms/step | 31.4 samples/s
|
| 1136 |
+
Step 41710/50462 | Loss: 2.4750 | LR: 3.73e-05 | 1913.3ms/step | 31.4 samples/s
|
| 1137 |
+
Step 41720/50462 | Loss: 2.4661 | LR: 3.72e-05 | 1914.7ms/step | 31.3 samples/s
|
| 1138 |
+
Step 41730/50462 | Loss: 2.4525 | LR: 3.71e-05 | 1916.1ms/step | 31.3 samples/s
|
| 1139 |
+
Step 41740/50462 | Loss: 2.4385 | LR: 3.70e-05 | 1917.0ms/step | 31.3 samples/s
|
| 1140 |
+
Step 41750/50462 | Loss: 2.4660 | LR: 3.69e-05 | 1914.6ms/step | 31.3 samples/s
|
| 1141 |
+
Validation at step 41750: Loss = 2.4998, Perplexity = 12.18
|
| 1142 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed70/run_20260823_043810/checkpoint-41750
|
| 1143 |
+
Step 41760/50462 | Loss: 2.4765 | LR: 3.69e-05 | 3279.4ms/step | 18.3 samples/s
|
| 1144 |
+
Step 41770/50462 | Loss: 2.4496 | LR: 3.68e-05 | 1898.5ms/step | 31.6 samples/s
|
| 1145 |
+
Step 41780/50462 | Loss: 2.4236 | LR: 3.67e-05 | 1906.4ms/step | 31.5 samples/s
|
| 1146 |
+
Step 41790/50462 | Loss: 2.4384 | LR: 3.66e-05 | 1906.8ms/step | 31.5 samples/s
|
| 1147 |
+
Step 41800/50462 | Loss: 2.4967 | LR: 3.65e-05 | 1907.0ms/step | 31.5 samples/s
|
| 1148 |
+
Step 41810/50462 | Loss: 2.4740 | LR: 3.64e-05 | 1911.2ms/step | 31.4 samples/s
|
| 1149 |
+
Step 41820/50462 | Loss: 2.4658 | LR: 3.64e-05 | 1912.3ms/step | 31.4 samples/s
|
| 1150 |
+
Step 41830/50462 | Loss: 2.4844 | LR: 3.63e-05 | 1913.2ms/step | 31.4 samples/s
|
| 1151 |
+
Step 41840/50462 | Loss: 2.4547 | LR: 3.62e-05 | 1913.2ms/step | 31.4 samples/s
|
| 1152 |
+
Step 41850/50462 | Loss: 2.4263 | LR: 3.61e-05 | 1908.4ms/step | 31.4 samples/s
|
| 1153 |
+
Step 41860/50462 | Loss: 2.4356 | LR: 3.60e-05 | 1909.9ms/step | 31.4 samples/s
|
| 1154 |
+
Step 41870/50462 | Loss: 2.4848 | LR: 3.60e-05 | 1912.4ms/step | 31.4 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu.log
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253",
|
| 16 |
+
"resume_from_checkpoint": null,
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 71,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed71 6gpu",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed71"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Initializing model from scratch (random weights)...
|
| 69 |
+
Checkpoint activations: False
|
| 70 |
+
Checkpoint activations: False
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
Checkpoint activations: False
|
| 74 |
+
Checkpoint activations: False
|
| 75 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
================================================================================
|
| 79 |
+
MODEL ARCHITECTURE
|
| 80 |
+
================================================================================
|
| 81 |
+
Model config:
|
| 82 |
+
vocab_size: 151936
|
| 83 |
+
hidden_size: 1536
|
| 84 |
+
num_hidden_layers: 28
|
| 85 |
+
num_attention_heads: 12
|
| 86 |
+
num_key_value_heads: 4
|
| 87 |
+
head_dim: 128
|
| 88 |
+
intermediate_size: 6144
|
| 89 |
+
max_position_embeddings: 32768
|
| 90 |
+
rope_theta: 1000000
|
| 91 |
+
rope_train_position_scale_set: None
|
| 92 |
+
attention_type: sparse_stochastic
|
| 93 |
+
sparse_block_size: 128
|
| 94 |
+
sparse_num_random_blocks: 31
|
| 95 |
+
sparse_mask_pool_size: 1
|
| 96 |
+
sparse_max_seq_length: 8192
|
| 97 |
+
|
| 98 |
+
Parameter count:
|
| 99 |
+
Total parameters: 1,202,352,640
|
| 100 |
+
Trainable parameters: 1,202,352,640
|
| 101 |
+
Non-trainable parameters: 0
|
| 102 |
+
Model size (BF16): 2.40 GB
|
| 103 |
+
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
LAYER-BY-LAYER SUMMARY
|
| 106 |
+
--------------------------------------------------------------------------------
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 109 |
+
================================================================================================================================================================
|
| 110 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 111 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 112 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 113 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 143 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 144 |
+
================================================================================================================================================================
|
| 145 |
+
Total params: 1,435,726,336
|
| 146 |
+
Trainable params: 1,435,726,336
|
| 147 |
+
Non-trainable params: 0
|
| 148 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 149 |
+
================================================================================================================================================================
|
| 150 |
+
Input size (MB): 0.07
|
| 151 |
+
Forward/backward pass size (MB): 26218.59
|
| 152 |
+
Params size (MB): 2871.45
|
| 153 |
+
Estimated Total Size (MB): 29090.11
|
| 154 |
+
================================================================================================================================================================
|
| 155 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 156 |
+
================================================================================
|
| 157 |
+
|
| 158 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 163 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 164 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 165 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 166 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 167 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 168 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 169 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 170 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 171 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 172 |
+
Dataset split: 3027764 train, 159356 validation
|
| 173 |
+
Training examples (packed, train split): 3027764
|
| 174 |
+
Validation examples (packed, val split): 159356
|
| 175 |
+
Per-device batch size: 1
|
| 176 |
+
World size (GPUs): 6
|
| 177 |
+
Gradient accumulation steps: 10
|
| 178 |
+
Effective batch size (seqs/step): 60
|
| 179 |
+
Sequence length: 8192
|
| 180 |
+
Tokens per optimizer step: 491,520
|
| 181 |
+
Micro-batches per epoch (per rank): 504628
|
| 182 |
+
Optimizer steps per epoch: 50462
|
| 183 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 184 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 185 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 186 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 187 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_140341-qctui00g
|
| 188 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 189 |
+
wandb: Syncing run 1.2b dense 8k seed71 6gpu
|
| 190 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 191 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qctui00g
|
| 192 |
+
Starting training...
|
| 193 |
+
Switching attention mode for step 1: dense (dense step interval)
|
| 194 |
+
Step 10/50462 | Loss: 12.1742 | LR: 6.67e-06 | 11731.3ms/step | 5.1 samples/s
|
| 195 |
+
Step 20/50462 | Loss: 11.8320 | LR: 1.33e-05 | 4686.4ms/step | 12.8 samples/s
|
| 196 |
+
Step 30/50462 | Loss: 10.8631 | LR: 2.00e-05 | 4685.8ms/step | 12.8 samples/s
|
| 197 |
+
Step 40/50462 | Loss: 10.0487 | LR: 2.67e-05 | 4683.0ms/step | 12.8 samples/s
|
| 198 |
+
Step 50/50462 | Loss: 9.6692 | LR: 3.33e-05 | 4685.5ms/step | 12.8 samples/s
|
| 199 |
+
Step 60/50462 | Loss: 9.2179 | LR: 4.00e-05 | 4687.5ms/step | 12.8 samples/s
|
| 200 |
+
Step 70/50462 | Loss: 8.6882 | LR: 4.67e-05 | 4694.4ms/step | 12.8 samples/s
|
| 201 |
+
Step 80/50462 | Loss: 8.2368 | LR: 5.33e-05 | 4694.2ms/step | 12.8 samples/s
|
| 202 |
+
Step 90/50462 | Loss: 7.8637 | LR: 6.00e-05 | 4696.7ms/step | 12.8 samples/s
|
| 203 |
+
Step 100/50462 | Loss: 7.5250 | LR: 6.67e-05 | 4698.3ms/step | 12.8 samples/s
|
| 204 |
+
Step 110/50462 | Loss: 7.2680 | LR: 7.33e-05 | 4700.0ms/step | 12.8 samples/s
|
| 205 |
+
Step 120/50462 | Loss: 7.1073 | LR: 8.00e-05 | 4701.1ms/step | 12.8 samples/s
|
| 206 |
+
Step 130/50462 | Loss: 6.9883 | LR: 8.67e-05 | 4704.2ms/step | 12.8 samples/s
|
| 207 |
+
Step 140/50462 | Loss: 6.9082 | LR: 9.33e-05 | 4706.6ms/step | 12.7 samples/s
|
| 208 |
+
Step 150/50462 | Loss: 6.7566 | LR: 1.00e-04 | 4705.5ms/step | 12.8 samples/s
|
| 209 |
+
Step 160/50462 | Loss: 6.7090 | LR: 1.07e-04 | 4707.2ms/step | 12.7 samples/s
|
| 210 |
+
Step 170/50462 | Loss: 6.5772 | LR: 1.13e-04 | 4706.8ms/step | 12.7 samples/s
|
| 211 |
+
Step 180/50462 | Loss: 6.4874 | LR: 1.20e-04 | 4709.3ms/step | 12.7 samples/s
|
| 212 |
+
Step 190/50462 | Loss: 6.3780 | LR: 1.27e-04 | 4710.0ms/step | 12.7 samples/s
|
| 213 |
+
Step 200/50462 | Loss: 6.3171 | LR: 1.33e-04 | 4710.6ms/step | 12.7 samples/s
|
| 214 |
+
Step 210/50462 | Loss: 6.2308 | LR: 1.40e-04 | 4711.4ms/step | 12.7 samples/s
|
| 215 |
+
Step 220/50462 | Loss: 6.2443 | LR: 1.47e-04 | 4711.4ms/step | 12.7 samples/s
|
| 216 |
+
Step 230/50462 | Loss: 6.1113 | LR: 1.53e-04 | 4711.8ms/step | 12.7 samples/s
|
| 217 |
+
Step 240/50462 | Loss: 6.0704 | LR: 1.60e-04 | 4713.0ms/step | 12.7 samples/s
|
| 218 |
+
Step 250/50462 | Loss: 6.0070 | LR: 1.67e-04 | 4713.5ms/step | 12.7 samples/s
|
| 219 |
+
Validation at step 250: Loss = 5.9502, Perplexity = 383.82
|
| 220 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-250
|
| 221 |
+
Step 260/50462 | Loss: 5.9161 | LR: 1.73e-04 | 10552.8ms/step | 5.7 samples/s
|
| 222 |
+
Step 270/50462 | Loss: 5.9269 | LR: 1.80e-04 | 4709.3ms/step | 12.7 samples/s
|
| 223 |
+
Step 280/50462 | Loss: 5.8761 | LR: 1.87e-04 | 4718.1ms/step | 12.7 samples/s
|
| 224 |
+
Step 290/50462 | Loss: 5.7764 | LR: 1.93e-04 | 4721.0ms/step | 12.7 samples/s
|
| 225 |
+
Step 300/50462 | Loss: 5.7620 | LR: 2.00e-04 | 4720.5ms/step | 12.7 samples/s
|
| 226 |
+
Step 310/50462 | Loss: 5.7145 | LR: 2.07e-04 | 4721.2ms/step | 12.7 samples/s
|
| 227 |
+
Step 320/50462 | Loss: 5.6582 | LR: 2.13e-04 | 4721.3ms/step | 12.7 samples/s
|
| 228 |
+
Step 330/50462 | Loss: 5.5893 | LR: 2.20e-04 | 4722.5ms/step | 12.7 samples/s
|
| 229 |
+
Step 340/50462 | Loss: 5.5291 | LR: 2.27e-04 | 4724.2ms/step | 12.7 samples/s
|
| 230 |
+
Step 350/50462 | Loss: 5.4462 | LR: 2.33e-04 | 4726.4ms/step | 12.7 samples/s
|
| 231 |
+
Step 360/50462 | Loss: 5.4114 | LR: 2.40e-04 | 4727.0ms/step | 12.7 samples/s
|
| 232 |
+
Step 370/50462 | Loss: 5.3658 | LR: 2.47e-04 | 4723.6ms/step | 12.7 samples/s
|
| 233 |
+
Step 380/50462 | Loss: 5.3126 | LR: 2.53e-04 | 4727.8ms/step | 12.7 samples/s
|
| 234 |
+
Step 390/50462 | Loss: 5.3079 | LR: 2.60e-04 | 4723.7ms/step | 12.7 samples/s
|
| 235 |
+
Step 400/50462 | Loss: 5.2380 | LR: 2.67e-04 | 4722.7ms/step | 12.7 samples/s
|
| 236 |
+
Step 410/50462 | Loss: 5.2100 | LR: 2.73e-04 | 4722.8ms/step | 12.7 samples/s
|
| 237 |
+
Step 420/50462 | Loss: 5.1419 | LR: 2.80e-04 | 4725.1ms/step | 12.7 samples/s
|
| 238 |
+
Step 430/50462 | Loss: 5.0929 | LR: 2.87e-04 | 4723.0ms/step | 12.7 samples/s
|
| 239 |
+
Step 440/50462 | Loss: 5.0683 | LR: 2.93e-04 | 4723.7ms/step | 12.7 samples/s
|
| 240 |
+
Step 450/50462 | Loss: 5.0881 | LR: 3.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 241 |
+
Step 460/50462 | Loss: 5.0190 | LR: 3.07e-04 | 4725.6ms/step | 12.7 samples/s
|
| 242 |
+
Step 470/50462 | Loss: 4.8955 | LR: 3.13e-04 | 4728.7ms/step | 12.7 samples/s
|
| 243 |
+
Step 480/50462 | Loss: 4.9059 | LR: 3.20e-04 | 4731.7ms/step | 12.7 samples/s
|
| 244 |
+
Step 490/50462 | Loss: 4.9201 | LR: 3.27e-04 | 4732.2ms/step | 12.7 samples/s
|
| 245 |
+
Step 500/50462 | Loss: 4.8885 | LR: 3.33e-04 | 4724.2ms/step | 12.7 samples/s
|
| 246 |
+
Validation at step 500: Loss = 4.8038, Perplexity = 121.97
|
| 247 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-500
|
| 248 |
+
Step 510/50462 | Loss: 4.7781 | LR: 3.40e-04 | 6965.7ms/step | 8.6 samples/s
|
| 249 |
+
Step 520/50462 | Loss: 4.8151 | LR: 3.47e-04 | 4710.8ms/step | 12.7 samples/s
|
| 250 |
+
Step 530/50462 | Loss: 4.8086 | LR: 3.53e-04 | 4716.2ms/step | 12.7 samples/s
|
| 251 |
+
Step 540/50462 | Loss: 4.7796 | LR: 3.60e-04 | 4721.8ms/step | 12.7 samples/s
|
| 252 |
+
Step 550/50462 | Loss: 4.7198 | LR: 3.67e-04 | 4726.1ms/step | 12.7 samples/s
|
| 253 |
+
Step 560/50462 | Loss: 4.6588 | LR: 3.73e-04 | 4725.2ms/step | 12.7 samples/s
|
| 254 |
+
Step 570/50462 | Loss: 4.6077 | LR: 3.80e-04 | 4726.1ms/step | 12.7 samples/s
|
| 255 |
+
Step 580/50462 | Loss: 4.6800 | LR: 3.87e-04 | 4727.3ms/step | 12.7 samples/s
|
| 256 |
+
Step 590/50462 | Loss: 4.5749 | LR: 3.93e-04 | 4726.6ms/step | 12.7 samples/s
|
| 257 |
+
Step 600/50462 | Loss: 4.5416 | LR: 4.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 258 |
+
Step 610/50462 | Loss: 4.5025 | LR: 4.07e-04 | 4725.2ms/step | 12.7 samples/s
|
| 259 |
+
Step 620/50462 | Loss: 4.5348 | LR: 4.13e-04 | 4723.7ms/step | 12.7 samples/s
|
| 260 |
+
Step 630/50462 | Loss: 4.5014 | LR: 4.20e-04 | 4724.4ms/step | 12.7 samples/s
|
| 261 |
+
Step 640/50462 | Loss: 4.3765 | LR: 4.27e-04 | 4724.0ms/step | 12.7 samples/s
|
| 262 |
+
Step 650/50462 | Loss: 4.3733 | LR: 4.33e-04 | 4728.0ms/step | 12.7 samples/s
|
| 263 |
+
Step 660/50462 | Loss: 4.4487 | LR: 4.40e-04 | 4729.0ms/step | 12.7 samples/s
|
| 264 |
+
Step 670/50462 | Loss: 4.3735 | LR: 4.47e-04 | 4724.5ms/step | 12.7 samples/s
|
| 265 |
+
Step 680/50462 | Loss: 4.2940 | LR: 4.53e-04 | 4724.8ms/step | 12.7 samples/s
|
| 266 |
+
Step 690/50462 | Loss: 4.2584 | LR: 4.60e-04 | 4725.7ms/step | 12.7 samples/s
|
| 267 |
+
Step 700/50462 | Loss: 4.2832 | LR: 4.67e-04 | 4725.6ms/step | 12.7 samples/s
|
| 268 |
+
Step 710/50462 | Loss: 4.2303 | LR: 4.73e-04 | 4724.7ms/step | 12.7 samples/s
|
| 269 |
+
Step 720/50462 | Loss: 4.2239 | LR: 4.80e-04 | 4724.2ms/step | 12.7 samples/s
|
| 270 |
+
Step 730/50462 | Loss: 4.2088 | LR: 4.87e-04 | 4725.5ms/step | 12.7 samples/s
|
| 271 |
+
Step 740/50462 | Loss: 4.1720 | LR: 4.93e-04 | 4725.0ms/step | 12.7 samples/s
|
| 272 |
+
Step 750/50462 | Loss: 4.0841 | LR: 5.00e-04 | 4727.0ms/step | 12.7 samples/s
|
| 273 |
+
Validation at step 750: Loss = 4.0713, Perplexity = 58.63
|
| 274 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-750
|
| 275 |
+
Step 760/50462 | Loss: 4.1001 | LR: 5.00e-04 | 6971.9ms/step | 8.6 samples/s
|
| 276 |
+
Step 770/50462 | Loss: 4.1039 | LR: 5.00e-04 | 4704.8ms/step | 12.8 samples/s
|
| 277 |
+
Step 780/50462 | Loss: 4.0593 | LR: 5.00e-04 | 4713.6ms/step | 12.7 samples/s
|
| 278 |
+
Step 790/50462 | Loss: 4.0365 | LR: 5.00e-04 | 4716.9ms/step | 12.7 samples/s
|
| 279 |
+
Step 800/50462 | Loss: 4.0207 | LR: 5.00e-04 | 4720.8ms/step | 12.7 samples/s
|
| 280 |
+
Step 810/50462 | Loss: 3.9969 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 281 |
+
Step 820/50462 | Loss: 3.9676 | LR: 5.00e-04 | 4720.4ms/step | 12.7 samples/s
|
| 282 |
+
Step 830/50462 | Loss: 3.9502 | LR: 5.00e-04 | 4721.6ms/step | 12.7 samples/s
|
| 283 |
+
Step 840/50462 | Loss: 3.8992 | LR: 5.00e-04 | 4719.3ms/step | 12.7 samples/s
|
| 284 |
+
Step 850/50462 | Loss: 3.8830 | LR: 5.00e-04 | 4719.8ms/step | 12.7 samples/s
|
| 285 |
+
Step 860/50462 | Loss: 3.8926 | LR: 5.00e-04 | 4720.0ms/step | 12.7 samples/s
|
| 286 |
+
Step 870/50462 | Loss: 3.8647 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 287 |
+
Step 880/50462 | Loss: 3.8662 | LR: 5.00e-04 | 4720.4ms/step | 12.7 samples/s
|
| 288 |
+
Step 890/50462 | Loss: 3.8305 | LR: 5.00e-04 | 4720.5ms/step | 12.7 samples/s
|
| 289 |
+
Step 900/50462 | Loss: 3.8499 | LR: 5.00e-04 | 4721.2ms/step | 12.7 samples/s
|
| 290 |
+
Step 910/50462 | Loss: 3.8032 | LR: 5.00e-04 | 4720.0ms/step | 12.7 samples/s
|
| 291 |
+
Step 920/50462 | Loss: 3.7715 | LR: 5.00e-04 | 4722.8ms/step | 12.7 samples/s
|
| 292 |
+
Step 930/50462 | Loss: 3.7964 | LR: 5.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 293 |
+
Step 940/50462 | Loss: 3.7777 | LR: 5.00e-04 | 4721.3ms/step | 12.7 samples/s
|
| 294 |
+
Step 950/50462 | Loss: 3.7519 | LR: 5.00e-04 | 4719.6ms/step | 12.7 samples/s
|
| 295 |
+
Step 960/50462 | Loss: 3.7624 | LR: 5.00e-04 | 4720.2ms/step | 12.7 samples/s
|
| 296 |
+
Step 970/50462 | Loss: 3.7137 | LR: 5.00e-04 | 4721.8ms/step | 12.7 samples/s
|
| 297 |
+
Step 980/50462 | Loss: 3.7370 | LR: 5.00e-04 | 4721.2ms/step | 12.7 samples/s
|
| 298 |
+
Step 990/50462 | Loss: 3.7066 | LR: 5.00e-04 | 4720.1ms/step | 12.7 samples/s
|
| 299 |
+
Step 1000/50462 | Loss: 3.6887 | LR: 5.00e-04 | 4720.9ms/step | 12.7 samples/s
|
| 300 |
+
Validation at step 1000: Loss = 3.6835, Perplexity = 39.78
|
| 301 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1000
|
| 302 |
+
Step 1010/50462 | Loss: 3.6802 | LR: 5.00e-04 | 6982.7ms/step | 8.6 samples/s
|
| 303 |
+
Step 1020/50462 | Loss: 3.6538 | LR: 5.00e-04 | 4709.8ms/step | 12.7 samples/s
|
| 304 |
+
Step 1030/50462 | Loss: 3.6368 | LR: 5.00e-04 | 4720.9ms/step | 12.7 samples/s
|
| 305 |
+
Step 1040/50462 | Loss: 3.6837 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 306 |
+
Step 1050/50462 | Loss: 3.6485 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 307 |
+
Step 1060/50462 | Loss: 3.6665 | LR: 5.00e-04 | 4725.5ms/step | 12.7 samples/s
|
| 308 |
+
Step 1070/50462 | Loss: 3.6531 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 309 |
+
Step 1080/50462 | Loss: 3.6400 | LR: 5.00e-04 | 4728.7ms/step | 12.7 samples/s
|
| 310 |
+
Step 1090/50462 | Loss: 3.5922 | LR: 5.00e-04 | 4725.9ms/step | 12.7 samples/s
|
| 311 |
+
Step 1100/50462 | Loss: 3.6015 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 312 |
+
Step 1110/50462 | Loss: 3.5827 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 313 |
+
Step 1120/50462 | Loss: 3.6256 | LR: 5.00e-04 | 4724.4ms/step | 12.7 samples/s
|
| 314 |
+
Step 1130/50462 | Loss: 3.5688 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 315 |
+
Step 1140/50462 | Loss: 3.5873 | LR: 5.00e-04 | 4725.2ms/step | 12.7 samples/s
|
| 316 |
+
Step 1150/50462 | Loss: 3.5207 | LR: 5.00e-04 | 4724.6ms/step | 12.7 samples/s
|
| 317 |
+
Step 1160/50462 | Loss: 3.5826 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 318 |
+
Step 1170/50462 | Loss: 3.5051 | LR: 5.00e-04 | 4728.4ms/step | 12.7 samples/s
|
| 319 |
+
Step 1180/50462 | Loss: 3.5318 | LR: 5.00e-04 | 4727.1ms/step | 12.7 samples/s
|
| 320 |
+
Step 1190/50462 | Loss: 3.5266 | LR: 5.00e-04 | 4726.0ms/step | 12.7 samples/s
|
| 321 |
+
Step 1200/50462 | Loss: 3.5324 | LR: 5.00e-04 | 4726.7ms/step | 12.7 samples/s
|
| 322 |
+
Step 1210/50462 | Loss: 3.5045 | LR: 5.00e-04 | 4726.1ms/step | 12.7 samples/s
|
| 323 |
+
Step 1220/50462 | Loss: 3.5279 | LR: 5.00e-04 | 4726.7ms/step | 12.7 samples/s
|
| 324 |
+
Step 1230/50462 | Loss: 3.5277 | LR: 5.00e-04 | 4727.0ms/step | 12.7 samples/s
|
| 325 |
+
Step 1240/50462 | Loss: 3.5033 | LR: 5.00e-04 | 4727.3ms/step | 12.7 samples/s
|
| 326 |
+
Step 1250/50462 | Loss: 3.4781 | LR: 5.00e-04 | 4725.7ms/step | 12.7 samples/s
|
| 327 |
+
Validation at step 1250: Loss = 3.4788, Perplexity = 32.42
|
| 328 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1250
|
| 329 |
+
Step 1260/50462 | Loss: 3.4758 | LR: 5.00e-04 | 6962.5ms/step | 8.6 samples/s
|
| 330 |
+
Step 1270/50462 | Loss: 3.4578 | LR: 5.00e-04 | 4710.7ms/step | 12.7 samples/s
|
| 331 |
+
Step 1280/50462 | Loss: 3.4830 | LR: 5.00e-04 | 4718.4ms/step | 12.7 samples/s
|
| 332 |
+
Step 1290/50462 | Loss: 3.5056 | LR: 5.00e-04 | 4725.4ms/step | 12.7 samples/s
|
| 333 |
+
Step 1300/50462 | Loss: 3.4777 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 334 |
+
Step 1310/50462 | Loss: 3.4519 | LR: 5.00e-04 | 4725.8ms/step | 12.7 samples/s
|
| 335 |
+
Step 1320/50462 | Loss: 3.4839 | LR: 5.00e-04 | 4726.3ms/step | 12.7 samples/s
|
| 336 |
+
Step 1330/50462 | Loss: 3.4750 | LR: 5.00e-04 | 4728.4ms/step | 12.7 samples/s
|
| 337 |
+
Step 1340/50462 | Loss: 3.4846 | LR: 5.00e-04 | 4726.6ms/step | 12.7 samples/s
|
| 338 |
+
Step 1350/50462 | Loss: 3.4618 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 339 |
+
Step 1360/50462 | Loss: 3.4365 | LR: 5.00e-04 | 4727.4ms/step | 12.7 samples/s
|
| 340 |
+
Step 1370/50462 | Loss: 3.4206 | LR: 5.00e-04 | 4726.1ms/step | 12.7 samples/s
|
| 341 |
+
Step 1380/50462 | Loss: 3.4555 | LR: 5.00e-04 | 4726.2ms/step | 12.7 samples/s
|
| 342 |
+
Step 1390/50462 | Loss: 3.4204 | LR: 5.00e-04 | 4727.3ms/step | 12.7 samples/s
|
| 343 |
+
Step 1400/50462 | Loss: 3.4113 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 344 |
+
Step 1410/50462 | Loss: 3.3973 | LR: 5.00e-04 | 4726.2ms/step | 12.7 samples/s
|
| 345 |
+
Step 1420/50462 | Loss: 3.3653 | LR: 5.00e-04 | 4725.4ms/step | 12.7 samples/s
|
| 346 |
+
Step 1430/50462 | Loss: 3.3901 | LR: 5.00e-04 | 4727.1ms/step | 12.7 samples/s
|
| 347 |
+
Step 1440/50462 | Loss: 3.3916 | LR: 5.00e-04 | 4727.2ms/step | 12.7 samples/s
|
| 348 |
+
Step 1450/50462 | Loss: 3.4192 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 349 |
+
Step 1460/50462 | Loss: 3.3773 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 350 |
+
Step 1470/50462 | Loss: 3.4135 | LR: 5.00e-04 | 4726.3ms/step | 12.7 samples/s
|
| 351 |
+
Step 1480/50462 | Loss: 3.3473 | LR: 5.00e-04 | 4728.3ms/step | 12.7 samples/s
|
| 352 |
+
Step 1490/50462 | Loss: 3.3635 | LR: 5.00e-04 | 4726.4ms/step | 12.7 samples/s
|
| 353 |
+
Step 1500/50462 | Loss: 3.3885 | LR: 5.00e-04 | 4726.5ms/step | 12.7 samples/s
|
| 354 |
+
Validation at step 1500: Loss = 3.3587, Perplexity = 28.75
|
| 355 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1500
|
| 356 |
+
Step 1510/50462 | Loss: 3.3174 | LR: 5.00e-04 | 6964.7ms/step | 8.6 samples/s
|
| 357 |
+
Step 1520/50462 | Loss: 3.3599 | LR: 5.00e-04 | 4705.3ms/step | 12.8 samples/s
|
| 358 |
+
Step 1530/50462 | Loss: 3.3518 | LR: 5.00e-04 | 4713.3ms/step | 12.7 samples/s
|
| 359 |
+
Step 1540/50462 | Loss: 3.3621 | LR: 5.00e-04 | 4718.4ms/step | 12.7 samples/s
|
| 360 |
+
Step 1550/50462 | Loss: 3.3820 | LR: 5.00e-04 | 4722.1ms/step | 12.7 samples/s
|
| 361 |
+
Step 1560/50462 | Loss: 3.3465 | LR: 5.00e-04 | 4721.6ms/step | 12.7 samples/s
|
| 362 |
+
Step 1570/50462 | Loss: 3.2939 | LR: 5.00e-04 | 4727.5ms/step | 12.7 samples/s
|
| 363 |
+
Step 1580/50462 | Loss: 3.3606 | LR: 5.00e-04 | 4725.3ms/step | 12.7 samples/s
|
| 364 |
+
Step 1590/50462 | Loss: 3.3297 | LR: 5.00e-04 | 4723.0ms/step | 12.7 samples/s
|
| 365 |
+
Step 1600/50462 | Loss: 3.3123 | LR: 5.00e-04 | 4726.4ms/step | 12.7 samples/s
|
| 366 |
+
Step 1610/50462 | Loss: 3.3400 | LR: 5.00e-04 | 4725.7ms/step | 12.7 samples/s
|
| 367 |
+
Step 1620/50462 | Loss: 3.3135 | LR: 5.00e-04 | 4724.1ms/step | 12.7 samples/s
|
| 368 |
+
Step 1630/50462 | Loss: 3.3332 | LR: 5.00e-04 | 4723.3ms/step | 12.7 samples/s
|
| 369 |
+
Step 1640/50462 | Loss: 3.3286 | LR: 5.00e-04 | 4723.5ms/step | 12.7 samples/s
|
| 370 |
+
Step 1650/50462 | Loss: 3.3389 | LR: 5.00e-04 | 4723.6ms/step | 12.7 samples/s
|
| 371 |
+
Step 1660/50462 | Loss: 3.2935 | LR: 5.00e-04 | 4723.7ms/step | 12.7 samples/s
|
| 372 |
+
Step 1670/50462 | Loss: 3.2991 | LR: 5.00e-04 | 4725.6ms/step | 12.7 samples/s
|
| 373 |
+
Step 1680/50462 | Loss: 3.2835 | LR: 5.00e-04 | 4723.9ms/step | 12.7 samples/s
|
| 374 |
+
Step 1690/50462 | Loss: 3.2968 | LR: 5.00e-04 | 4723.8ms/step | 12.7 samples/s
|
| 375 |
+
Step 1700/50462 | Loss: 3.2718 | LR: 5.00e-04 | 4725.1ms/step | 12.7 samples/s
|
| 376 |
+
Step 1710/50462 | Loss: 3.3012 | LR: 5.00e-04 | 4726.8ms/step | 12.7 samples/s
|
| 377 |
+
Step 1720/50462 | Loss: 3.2430 | LR: 5.00e-04 | 4728.1ms/step | 12.7 samples/s
|
| 378 |
+
Step 1730/50462 | Loss: 3.2939 | LR: 5.00e-04 | 4729.9ms/step | 12.7 samples/s
|
| 379 |
+
Step 1740/50462 | Loss: 3.2910 | LR: 5.00e-04 | 4728.6ms/step | 12.7 samples/s
|
| 380 |
+
Step 1750/50462 | Loss: 3.2874 | LR: 5.00e-04 | 4726.8ms/step | 12.7 samples/s
|
| 381 |
+
Validation at step 1750: Loss = 3.2720, Perplexity = 26.36
|
| 382 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-1750
|
| 383 |
+
Step 1760/50462 | Loss: 3.3130 | LR: 4.99e-04 | 7002.9ms/step | 8.6 samples/s
|
| 384 |
+
Step 1770/50462 | Loss: 3.2957 | LR: 4.99e-04 | 4715.3ms/step | 12.7 samples/s
|
| 385 |
+
Step 1780/50462 | Loss: 3.2438 | LR: 4.99e-04 | 4724.5ms/step | 12.7 samples/s
|
| 386 |
+
Step 1790/50462 | Loss: 3.2295 | LR: 4.99e-04 | 4729.4ms/step | 12.7 samples/s
|
| 387 |
+
Step 1800/50462 | Loss: 3.2176 | LR: 4.99e-04 | 4730.0ms/step | 12.7 samples/s
|
| 388 |
+
Step 1810/50462 | Loss: 3.2666 | LR: 4.99e-04 | 4729.2ms/step | 12.7 samples/s
|
| 389 |
+
Step 1820/50462 | Loss: 3.2809 | LR: 4.99e-04 | 4729.6ms/step | 12.7 samples/s
|
| 390 |
+
Step 1830/50462 | Loss: 3.2583 | LR: 4.99e-04 | 4733.9ms/step | 12.7 samples/s
|
| 391 |
+
Step 1840/50462 | Loss: 3.2340 | LR: 4.99e-04 | 4730.8ms/step | 12.7 samples/s
|
| 392 |
+
Step 1850/50462 | Loss: 3.1894 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 393 |
+
Step 1860/50462 | Loss: 3.2571 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 394 |
+
Step 1870/50462 | Loss: 3.2353 | LR: 4.99e-04 | 4727.9ms/step | 12.7 samples/s
|
| 395 |
+
Step 1880/50462 | Loss: 3.2044 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 396 |
+
Step 1890/50462 | Loss: 3.2278 | LR: 4.99e-04 | 4729.9ms/step | 12.7 samples/s
|
| 397 |
+
Step 1900/50462 | Loss: 3.3148 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 398 |
+
Step 1910/50462 | Loss: 3.2301 | LR: 4.99e-04 | 4729.8ms/step | 12.7 samples/s
|
| 399 |
+
Step 1920/50462 | Loss: 3.2385 | LR: 4.99e-04 | 4731.7ms/step | 12.7 samples/s
|
| 400 |
+
Step 1930/50462 | Loss: 3.2242 | LR: 4.99e-04 | 4730.1ms/step | 12.7 samples/s
|
| 401 |
+
Step 1940/50462 | Loss: 3.2372 | LR: 4.99e-04 | 4729.3ms/step | 12.7 samples/s
|
| 402 |
+
Step 1950/50462 | Loss: 3.2175 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 403 |
+
Step 1960/50462 | Loss: 3.2014 | LR: 4.99e-04 | 4729.8ms/step | 12.7 samples/s
|
| 404 |
+
Step 1970/50462 | Loss: 3.2542 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 405 |
+
Step 1980/50462 | Loss: 3.2235 | LR: 4.99e-04 | 4732.0ms/step | 12.7 samples/s
|
| 406 |
+
Step 1990/50462 | Loss: 3.2450 | LR: 4.99e-04 | 4731.8ms/step | 12.7 samples/s
|
| 407 |
+
Step 2000/50462 | Loss: 3.2124 | LR: 4.99e-04 | 4733.8ms/step | 12.7 samples/s
|
| 408 |
+
Validation at step 2000: Loss = 3.2086, Perplexity = 24.74
|
| 409 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2000
|
| 410 |
+
Step 2010/50462 | Loss: 3.2572 | LR: 4.99e-04 | 6993.8ms/step | 8.6 samples/s
|
| 411 |
+
Step 2020/50462 | Loss: 3.2106 | LR: 4.99e-04 | 4715.2ms/step | 12.7 samples/s
|
| 412 |
+
Step 2030/50462 | Loss: 3.2275 | LR: 4.99e-04 | 4719.9ms/step | 12.7 samples/s
|
| 413 |
+
Step 2040/50462 | Loss: 3.2156 | LR: 4.99e-04 | 4724.1ms/step | 12.7 samples/s
|
| 414 |
+
Step 2050/50462 | Loss: 3.2348 | LR: 4.99e-04 | 4724.6ms/step | 12.7 samples/s
|
| 415 |
+
Step 2060/50462 | Loss: 3.2143 | LR: 4.99e-04 | 4726.6ms/step | 12.7 samples/s
|
| 416 |
+
Step 2070/50462 | Loss: 3.2341 | LR: 4.99e-04 | 4726.1ms/step | 12.7 samples/s
|
| 417 |
+
Step 2080/50462 | Loss: 3.2059 | LR: 4.99e-04 | 4726.9ms/step | 12.7 samples/s
|
| 418 |
+
Step 2090/50462 | Loss: 3.1874 | LR: 4.99e-04 | 4725.4ms/step | 12.7 samples/s
|
| 419 |
+
Step 2100/50462 | Loss: 3.1995 | LR: 4.99e-04 | 4728.1ms/step | 12.7 samples/s
|
| 420 |
+
Step 2110/50462 | Loss: 3.1995 | LR: 4.99e-04 | 4725.9ms/step | 12.7 samples/s
|
| 421 |
+
Step 2120/50462 | Loss: 3.1980 | LR: 4.99e-04 | 4725.7ms/step | 12.7 samples/s
|
| 422 |
+
Step 2130/50462 | Loss: 3.1753 | LR: 4.99e-04 | 4725.2ms/step | 12.7 samples/s
|
| 423 |
+
Step 2140/50462 | Loss: 3.1705 | LR: 4.99e-04 | 4727.7ms/step | 12.7 samples/s
|
| 424 |
+
Step 2150/50462 | Loss: 3.1975 | LR: 4.99e-04 | 4727.7ms/step | 12.7 samples/s
|
| 425 |
+
Step 2160/50462 | Loss: 3.2377 | LR: 4.99e-04 | 4725.5ms/step | 12.7 samples/s
|
| 426 |
+
Step 2170/50462 | Loss: 3.1811 | LR: 4.99e-04 | 4725.1ms/step | 12.7 samples/s
|
| 427 |
+
Step 2180/50462 | Loss: 3.1681 | LR: 4.99e-04 | 4726.8ms/step | 12.7 samples/s
|
| 428 |
+
Step 2190/50462 | Loss: 3.1791 | LR: 4.99e-04 | 4727.1ms/step | 12.7 samples/s
|
| 429 |
+
Step 2200/50462 | Loss: 3.1861 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 430 |
+
Step 2210/50462 | Loss: 3.1742 | LR: 4.99e-04 | 4726.3ms/step | 12.7 samples/s
|
| 431 |
+
Step 2220/50462 | Loss: 3.1546 | LR: 4.99e-04 | 4727.0ms/step | 12.7 samples/s
|
| 432 |
+
Step 2230/50462 | Loss: 3.1727 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 433 |
+
Step 2240/50462 | Loss: 3.1488 | LR: 4.99e-04 | 4725.7ms/step | 12.7 samples/s
|
| 434 |
+
Step 2250/50462 | Loss: 3.1842 | LR: 4.99e-04 | 4726.7ms/step | 12.7 samples/s
|
| 435 |
+
Validation at step 2250: Loss = 3.1563, Perplexity = 23.48
|
| 436 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2250
|
| 437 |
+
Step 2260/50462 | Loss: 3.1741 | LR: 4.99e-04 | 6990.2ms/step | 8.6 samples/s
|
| 438 |
+
Step 2270/50462 | Loss: 3.1603 | LR: 4.99e-04 | 4711.9ms/step | 12.7 samples/s
|
| 439 |
+
Step 2280/50462 | Loss: 3.1767 | LR: 4.99e-04 | 4720.4ms/step | 12.7 samples/s
|
| 440 |
+
Step 2290/50462 | Loss: 3.1850 | LR: 4.99e-04 | 4725.3ms/step | 12.7 samples/s
|
| 441 |
+
Step 2300/50462 | Loss: 3.1567 | LR: 4.99e-04 | 4727.0ms/step | 12.7 samples/s
|
| 442 |
+
Step 2310/50462 | Loss: 3.1578 | LR: 4.99e-04 | 4726.9ms/step | 12.7 samples/s
|
| 443 |
+
Step 2320/50462 | Loss: 3.1718 | LR: 4.99e-04 | 4727.6ms/step | 12.7 samples/s
|
| 444 |
+
Step 2330/50462 | Loss: 3.1383 | LR: 4.99e-04 | 4727.4ms/step | 12.7 samples/s
|
| 445 |
+
Step 2340/50462 | Loss: 3.1241 | LR: 4.99e-04 | 4727.4ms/step | 12.7 samples/s
|
| 446 |
+
Step 2350/50462 | Loss: 3.0744 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 447 |
+
Step 2360/50462 | Loss: 3.1817 | LR: 4.99e-04 | 4728.2ms/step | 12.7 samples/s
|
| 448 |
+
Step 2370/50462 | Loss: 3.1758 | LR: 4.99e-04 | 4729.2ms/step | 12.7 samples/s
|
| 449 |
+
Step 2380/50462 | Loss: 3.1503 | LR: 4.99e-04 | 4728.7ms/step | 12.7 samples/s
|
| 450 |
+
Step 2390/50462 | Loss: 3.0959 | LR: 4.99e-04 | 4728.8ms/step | 12.7 samples/s
|
| 451 |
+
Step 2400/50462 | Loss: 3.1322 | LR: 4.99e-04 | 4730.2ms/step | 12.7 samples/s
|
| 452 |
+
Step 2410/50462 | Loss: 3.1433 | LR: 4.99e-04 | 4728.3ms/step | 12.7 samples/s
|
| 453 |
+
Step 2420/50462 | Loss: 3.1539 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 454 |
+
Step 2430/50462 | Loss: 3.1273 | LR: 4.99e-04 | 4728.7ms/step | 12.7 samples/s
|
| 455 |
+
Step 2440/50462 | Loss: 3.1468 | LR: 4.99e-04 | 4731.2ms/step | 12.7 samples/s
|
| 456 |
+
Step 2450/50462 | Loss: 3.1141 | LR: 4.99e-04 | 4730.5ms/step | 12.7 samples/s
|
| 457 |
+
Step 2460/50462 | Loss: 3.0949 | LR: 4.99e-04 | 4729.4ms/step | 12.7 samples/s
|
| 458 |
+
Step 2470/50462 | Loss: 3.1297 | LR: 4.99e-04 | 4730.3ms/step | 12.7 samples/s
|
| 459 |
+
Step 2480/50462 | Loss: 3.0989 | LR: 4.99e-04 | 4729.9ms/step | 12.7 samples/s
|
| 460 |
+
Step 2490/50462 | Loss: 3.1122 | LR: 4.98e-04 | 4729.0ms/step | 12.7 samples/s
|
| 461 |
+
Step 2500/50462 | Loss: 3.1597 | LR: 4.98e-04 | 4728.5ms/step | 12.7 samples/s
|
| 462 |
+
Validation at step 2500: Loss = 3.1156, Perplexity = 22.55
|
| 463 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-2500
|
| 464 |
+
Step 2510/50462 | Loss: 3.0855 | LR: 4.98e-04 | 6997.4ms/step | 8.6 samples/s
|
| 465 |
+
Step 2520/50462 | Loss: 3.1177 | LR: 4.98e-04 | 4715.5ms/step | 12.7 samples/s
|
| 466 |
+
Step 2530/50462 | Loss: 3.1661 | LR: 4.98e-04 | 4723.3ms/step | 12.7 samples/s
|
| 467 |
+
Step 2540/50462 | Loss: 3.1501 | LR: 4.98e-04 | 4724.6ms/step | 12.7 samples/s
|
| 468 |
+
Step 2550/50462 | Loss: 3.1158 | LR: 4.98e-04 | 4724.0ms/step | 12.7 samples/s
|
| 469 |
+
Step 2560/50462 | Loss: 3.0982 | LR: 4.98e-04 | 4725.0ms/step | 12.7 samples/s
|
| 470 |
+
Step 2570/50462 | Loss: 3.1328 | LR: 4.98e-04 | 4722.0ms/step | 12.7 samples/s
|
| 471 |
+
Step 2580/50462 | Loss: 3.0830 | LR: 4.98e-04 | 4723.9ms/step | 12.7 samples/s
|
| 472 |
+
Step 2590/50462 | Loss: 3.1025 | LR: 4.98e-04 | 4723.8ms/step | 12.7 samples/s
|
| 473 |
+
Step 2600/50462 | Loss: 3.1344 | LR: 4.98e-04 | 4723.3ms/step | 12.7 samples/s
|
| 474 |
+
Step 2610/50462 | Loss: 3.0735 | LR: 4.98e-04 | 4722.8ms/step | 12.7 samples/s
|
| 475 |
+
Step 2620/50462 | Loss: 3.1237 | LR: 4.98e-04 | 4723.9ms/step | 12.7 samples/s
|
| 476 |
+
Step 2630/50462 | Loss: 3.1402 | LR: 4.98e-04 | 4725.5ms/step | 12.7 samples/s
|
epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume2.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_dense_seed71_6gpu_resume_latest.log
ADDED
|
@@ -0,0 +1,723 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0826 12:48:51.674000 237 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0826 12:48:51.674000 237 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0826 12:48:51.674000 237 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0826 12:48:51.674000 237 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46000",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 71,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": false,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b dense 8k seed71 6gpu",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"dense",
|
| 45 |
+
"seed71"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": 28,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 31,
|
| 58 |
+
"sparse_mask_pool_size": 1,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": null,
|
| 62 |
+
"sparse_mask_seed": 0,
|
| 63 |
+
"sparse_mixed_nrb_set": null,
|
| 64 |
+
"dense_mode": true,
|
| 65 |
+
"dense_step_intervals": null
|
| 66 |
+
}
|
| 67 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 68 |
+
Checkpoint activations: FalseCheckpoint activations: FalseResuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46000
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
Checkpoint activations: False
|
| 72 |
+
Checkpoint activations: False
|
| 73 |
+
Checkpoint activations: False
|
| 74 |
+
Checkpoint activations: False
|
| 75 |
+
dense_mode=True: sparse attention layers will use standard causal SDPA.
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
================================================================================
|
| 79 |
+
MODEL ARCHITECTURE
|
| 80 |
+
================================================================================
|
| 81 |
+
Model config:
|
| 82 |
+
vocab_size: 151936
|
| 83 |
+
hidden_size: 1536
|
| 84 |
+
num_hidden_layers: 28
|
| 85 |
+
num_attention_heads: 12
|
| 86 |
+
num_key_value_heads: 4
|
| 87 |
+
head_dim: 128
|
| 88 |
+
intermediate_size: 6144
|
| 89 |
+
max_position_embeddings: 32768
|
| 90 |
+
rope_theta: 1000000
|
| 91 |
+
rope_train_position_scale_set: None
|
| 92 |
+
attention_type: sparse_stochastic
|
| 93 |
+
sparse_block_size: 128
|
| 94 |
+
sparse_num_random_blocks: 31
|
| 95 |
+
sparse_mask_pool_size: 1
|
| 96 |
+
sparse_max_seq_length: 8192
|
| 97 |
+
|
| 98 |
+
Parameter count:
|
| 99 |
+
Total parameters: 1,202,352,640
|
| 100 |
+
Trainable parameters: 1,202,352,640
|
| 101 |
+
Non-trainable parameters: 0
|
| 102 |
+
Model size (BF16): 2.40 GB
|
| 103 |
+
|
| 104 |
+
--------------------------------------------------------------------------------
|
| 105 |
+
LAYER-BY-LAYER SUMMARY
|
| 106 |
+
--------------------------------------------------------------------------------
|
| 107 |
+
================================================================================================================================================================
|
| 108 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 109 |
+
================================================================================================================================================================
|
| 110 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 111 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 112 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 113 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 114 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 115 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 116 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 117 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 118 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 119 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 120 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 121 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 122 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 123 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 143 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 144 |
+
================================================================================================================================================================
|
| 145 |
+
Total params: 1,435,726,336
|
| 146 |
+
Trainable params: 1,435,726,336
|
| 147 |
+
Non-trainable params: 0
|
| 148 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 149 |
+
================================================================================================================================================================
|
| 150 |
+
Input size (MB): 0.07
|
| 151 |
+
Forward/backward pass size (MB): 26218.59
|
| 152 |
+
Params size (MB): 2871.45
|
| 153 |
+
Estimated Total Size (MB): 29090.11
|
| 154 |
+
================================================================================================================================================================
|
| 155 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 156 |
+
================================================================================
|
| 157 |
+
|
| 158 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 159 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 160 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 161 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 162 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 163 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 164 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 165 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 166 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 167 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 168 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 169 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 170 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 171 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 172 |
+
Dataset split: 3027764 train, 159356 validation
|
| 173 |
+
Training examples (packed, train split): 3027764
|
| 174 |
+
Validation examples (packed, val split): 159356
|
| 175 |
+
Per-device batch size: 1
|
| 176 |
+
World size (GPUs): 6
|
| 177 |
+
Gradient accumulation steps: 10
|
| 178 |
+
Effective batch size (seqs/step): 60
|
| 179 |
+
Sequence length: 8192
|
| 180 |
+
Tokens per optimizer step: 491,520
|
| 181 |
+
Micro-batches per epoch (per rank): 504628
|
| 182 |
+
Optimizer steps per epoch: 50462
|
| 183 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 184 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46000...
|
| 185 |
+
Preserving checkpoint LR state for exact resume: current=[9.873402670642934e-06], schedule_base=[0.0005]
|
| 186 |
+
Resumed from checkpoint at step 46000 (epoch 0, micro_step 0)
|
| 187 |
+
Resuming training from step 46000
|
| 188 |
+
Resuming wandb run qctui00g
|
| 189 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 190 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 191 |
+
wandb: setting up run qctui00g
|
| 192 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 193 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_125030-qctui00g
|
| 194 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 195 |
+
wandb: Resuming run 1.2b dense 8k seed71 6gpu
|
| 196 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 197 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qctui00g
|
| 198 |
+
Starting training...
|
| 199 |
+
Validation at step 46000: Loss = 2.4901, Perplexity = 12.06
|
| 200 |
+
Switching attention mode for step 46001: dense (dense step interval)
|
| 201 |
+
wandb: WARNING Tried to log to step 46000 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 202 |
+
Step 46010/50462 | Loss: 2.4781 | LR: 9.83e-06 | 10594.7ms/step | 5.7 samples/s
|
| 203 |
+
wandb: WARNING Tried to log to step 46010 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 204 |
+
Step 46020/50462 | Loss: 2.4680 | LR: 9.79e-06 | 1885.5ms/step | 31.8 samples/s
|
| 205 |
+
wandb: WARNING Tried to log to step 46020 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 206 |
+
Step 46030/50462 | Loss: 2.4924 | LR: 9.74e-06 | 1892.8ms/step | 31.7 samples/s
|
| 207 |
+
wandb: WARNING Tried to log to step 46030 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 208 |
+
Step 46040/50462 | Loss: 2.5066 | LR: 9.70e-06 | 1899.2ms/step | 31.6 samples/s
|
| 209 |
+
wandb: WARNING Tried to log to step 46040 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Step 46050/50462 | Loss: 2.4522 | LR: 9.65e-06 | 1898.7ms/step | 31.6 samples/s
|
| 211 |
+
wandb: WARNING Tried to log to step 46050 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 212 |
+
Step 46060/50462 | Loss: 2.5047 | LR: 9.61e-06 | 1903.3ms/step | 31.5 samples/s
|
| 213 |
+
wandb: WARNING Tried to log to step 46060 that is less than the current step 46061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Step 46070/50462 | Loss: 2.5080 | LR: 9.57e-06 | 1905.1ms/step | 31.5 samples/s
|
| 215 |
+
Step 46080/50462 | Loss: 2.4660 | LR: 9.52e-06 | 1903.7ms/step | 31.5 samples/s
|
| 216 |
+
Step 46090/50462 | Loss: 2.4806 | LR: 9.48e-06 | 1906.1ms/step | 31.5 samples/s
|
| 217 |
+
Step 46100/50462 | Loss: 2.4616 | LR: 9.44e-06 | 1911.5ms/step | 31.4 samples/s
|
| 218 |
+
Step 46110/50462 | Loss: 2.4915 | LR: 9.40e-06 | 1911.6ms/step | 31.4 samples/s
|
| 219 |
+
Step 46120/50462 | Loss: 2.4639 | LR: 9.35e-06 | 1910.4ms/step | 31.4 samples/s
|
| 220 |
+
Step 46130/50462 | Loss: 2.4625 | LR: 9.31e-06 | 1911.2ms/step | 31.4 samples/s
|
| 221 |
+
Step 46140/50462 | Loss: 2.4934 | LR: 9.27e-06 | 1911.8ms/step | 31.4 samples/s
|
| 222 |
+
Step 46150/50462 | Loss: 2.4476 | LR: 9.22e-06 | 1915.3ms/step | 31.3 samples/s
|
| 223 |
+
Step 46160/50462 | Loss: 2.5042 | LR: 9.18e-06 | 1914.6ms/step | 31.3 samples/s
|
| 224 |
+
Step 46170/50462 | Loss: 2.5050 | LR: 9.14e-06 | 1918.2ms/step | 31.3 samples/s
|
| 225 |
+
Step 46180/50462 | Loss: 2.4913 | LR: 9.10e-06 | 1903.1ms/step | 31.5 samples/s
|
| 226 |
+
Step 46190/50462 | Loss: 2.4717 | LR: 9.06e-06 | 1904.5ms/step | 31.5 samples/s
|
| 227 |
+
Step 46200/50462 | Loss: 2.4604 | LR: 9.01e-06 | 1907.1ms/step | 31.5 samples/s
|
| 228 |
+
Step 46210/50462 | Loss: 2.4437 | LR: 8.97e-06 | 1905.6ms/step | 31.5 samples/s
|
| 229 |
+
Step 46220/50462 | Loss: 2.4667 | LR: 8.93e-06 | 1903.6ms/step | 31.5 samples/s
|
| 230 |
+
Step 46230/50462 | Loss: 2.4739 | LR: 8.89e-06 | 1906.0ms/step | 31.5 samples/s
|
| 231 |
+
Step 46240/50462 | Loss: 2.4816 | LR: 8.85e-06 | 1908.1ms/step | 31.4 samples/s
|
| 232 |
+
Step 46250/50462 | Loss: 2.4440 | LR: 8.80e-06 | 1913.6ms/step | 31.4 samples/s
|
| 233 |
+
Validation at step 46250: Loss = 2.4901, Perplexity = 12.06
|
| 234 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46250
|
| 235 |
+
Step 46260/50462 | Loss: 2.4454 | LR: 8.76e-06 | 3499.6ms/step | 17.1 samples/s
|
| 236 |
+
Step 46270/50462 | Loss: 2.4779 | LR: 8.72e-06 | 1903.6ms/step | 31.5 samples/s
|
| 237 |
+
Step 46280/50462 | Loss: 2.4665 | LR: 8.68e-06 | 1907.5ms/step | 31.5 samples/s
|
| 238 |
+
Step 46290/50462 | Loss: 2.4677 | LR: 8.64e-06 | 1913.6ms/step | 31.4 samples/s
|
| 239 |
+
Step 46300/50462 | Loss: 2.5092 | LR: 8.60e-06 | 1916.4ms/step | 31.3 samples/s
|
| 240 |
+
Step 46310/50462 | Loss: 2.5264 | LR: 8.56e-06 | 1912.1ms/step | 31.4 samples/s
|
| 241 |
+
Step 46320/50462 | Loss: 2.5250 | LR: 8.52e-06 | 1912.1ms/step | 31.4 samples/s
|
| 242 |
+
Step 46330/50462 | Loss: 2.4992 | LR: 8.47e-06 | 1914.9ms/step | 31.3 samples/s
|
| 243 |
+
Step 46340/50462 | Loss: 2.4891 | LR: 8.43e-06 | 1914.8ms/step | 31.3 samples/s
|
| 244 |
+
Step 46350/50462 | Loss: 2.4781 | LR: 8.39e-06 | 1913.8ms/step | 31.4 samples/s
|
| 245 |
+
Step 46360/50462 | Loss: 2.4871 | LR: 8.35e-06 | 1916.1ms/step | 31.3 samples/s
|
| 246 |
+
Step 46370/50462 | Loss: 2.4626 | LR: 8.31e-06 | 1918.0ms/step | 31.3 samples/s
|
| 247 |
+
Step 46380/50462 | Loss: 2.4759 | LR: 8.27e-06 | 1917.7ms/step | 31.3 samples/s
|
| 248 |
+
Step 46390/50462 | Loss: 2.4868 | LR: 8.23e-06 | 1920.8ms/step | 31.2 samples/s
|
| 249 |
+
Step 46400/50462 | Loss: 2.5023 | LR: 8.19e-06 | 1916.4ms/step | 31.3 samples/s
|
| 250 |
+
Step 46410/50462 | Loss: 2.4797 | LR: 8.15e-06 | 1920.1ms/step | 31.2 samples/s
|
| 251 |
+
Step 46420/50462 | Loss: 2.4761 | LR: 8.11e-06 | 1918.6ms/step | 31.3 samples/s
|
| 252 |
+
Step 46430/50462 | Loss: 2.4952 | LR: 8.07e-06 | 1918.9ms/step | 31.3 samples/s
|
| 253 |
+
Step 46440/50462 | Loss: 2.4888 | LR: 8.03e-06 | 1920.0ms/step | 31.2 samples/s
|
| 254 |
+
Step 46450/50462 | Loss: 2.5052 | LR: 7.99e-06 | 1922.2ms/step | 31.2 samples/s
|
| 255 |
+
Step 46460/50462 | Loss: 2.4744 | LR: 7.95e-06 | 1920.5ms/step | 31.2 samples/s
|
| 256 |
+
Step 46470/50462 | Loss: 2.4405 | LR: 7.91e-06 | 1911.8ms/step | 31.4 samples/s
|
| 257 |
+
Step 46480/50462 | Loss: 2.4470 | LR: 7.87e-06 | 1907.2ms/step | 31.5 samples/s
|
| 258 |
+
Step 46490/50462 | Loss: 2.4695 | LR: 7.83e-06 | 1914.2ms/step | 31.3 samples/s
|
| 259 |
+
Step 46500/50462 | Loss: 2.4940 | LR: 7.80e-06 | 1914.0ms/step | 31.3 samples/s
|
| 260 |
+
Validation at step 46500: Loss = 2.4901, Perplexity = 12.06
|
| 261 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46500
|
| 262 |
+
Step 46510/50462 | Loss: 2.4600 | LR: 7.76e-06 | 3333.7ms/step | 18.0 samples/s
|
| 263 |
+
Step 46520/50462 | Loss: 2.4911 | LR: 7.72e-06 | 1902.4ms/step | 31.5 samples/s
|
| 264 |
+
Step 46530/50462 | Loss: 2.4717 | LR: 7.68e-06 | 1906.7ms/step | 31.5 samples/s
|
| 265 |
+
Step 46540/50462 | Loss: 2.4302 | LR: 7.64e-06 | 1910.2ms/step | 31.4 samples/s
|
| 266 |
+
Step 46550/50462 | Loss: 2.4688 | LR: 7.60e-06 | 1910.3ms/step | 31.4 samples/s
|
| 267 |
+
Step 46560/50462 | Loss: 2.4821 | LR: 7.56e-06 | 1914.1ms/step | 31.3 samples/s
|
| 268 |
+
Step 46570/50462 | Loss: 2.4456 | LR: 7.52e-06 | 1907.4ms/step | 31.5 samples/s
|
| 269 |
+
Step 46580/50462 | Loss: 2.4971 | LR: 7.49e-06 | 1909.2ms/step | 31.4 samples/s
|
| 270 |
+
Step 46590/50462 | Loss: 2.4451 | LR: 7.45e-06 | 1907.2ms/step | 31.5 samples/s
|
| 271 |
+
Step 46600/50462 | Loss: 2.4968 | LR: 7.41e-06 | 1907.8ms/step | 31.5 samples/s
|
| 272 |
+
Step 46610/50462 | Loss: 2.4613 | LR: 7.37e-06 | 1910.0ms/step | 31.4 samples/s
|
| 273 |
+
Step 46620/50462 | Loss: 2.4447 | LR: 7.33e-06 | 1912.0ms/step | 31.4 samples/s
|
| 274 |
+
Step 46630/50462 | Loss: 2.4910 | LR: 7.29e-06 | 1912.0ms/step | 31.4 samples/s
|
| 275 |
+
Step 46640/50462 | Loss: 2.4336 | LR: 7.26e-06 | 1914.8ms/step | 31.3 samples/s
|
| 276 |
+
Step 46650/50462 | Loss: 2.5004 | LR: 7.22e-06 | 1915.3ms/step | 31.3 samples/s
|
| 277 |
+
Step 46660/50462 | Loss: 2.4664 | LR: 7.18e-06 | 1913.8ms/step | 31.4 samples/s
|
| 278 |
+
Step 46670/50462 | Loss: 2.4623 | LR: 7.14e-06 | 1910.5ms/step | 31.4 samples/s
|
| 279 |
+
Step 46680/50462 | Loss: 2.4967 | LR: 7.11e-06 | 1910.9ms/step | 31.4 samples/s
|
| 280 |
+
Step 46690/50462 | Loss: 2.4711 | LR: 7.07e-06 | 1914.0ms/step | 31.3 samples/s
|
| 281 |
+
Step 46700/50462 | Loss: 2.4935 | LR: 7.03e-06 | 1914.9ms/step | 31.3 samples/s
|
| 282 |
+
Step 46710/50462 | Loss: 2.4416 | LR: 6.99e-06 | 1911.1ms/step | 31.4 samples/s
|
| 283 |
+
Step 46720/50462 | Loss: 2.5056 | LR: 6.96e-06 | 1907.4ms/step | 31.5 samples/s
|
| 284 |
+
Step 46730/50462 | Loss: 2.4973 | LR: 6.92e-06 | 1913.3ms/step | 31.4 samples/s
|
| 285 |
+
Step 46740/50462 | Loss: 2.4861 | LR: 6.88e-06 | 1913.3ms/step | 31.4 samples/s
|
| 286 |
+
Step 46750/50462 | Loss: 2.4302 | LR: 6.85e-06 | 1910.7ms/step | 31.4 samples/s
|
| 287 |
+
Validation at step 46750: Loss = 2.4901, Perplexity = 12.06
|
| 288 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-46750
|
| 289 |
+
Step 46760/50462 | Loss: 2.4814 | LR: 6.81e-06 | 3320.3ms/step | 18.1 samples/s
|
| 290 |
+
Step 46770/50462 | Loss: 2.4945 | LR: 6.77e-06 | 1898.7ms/step | 31.6 samples/s
|
| 291 |
+
Step 46780/50462 | Loss: 2.4930 | LR: 6.74e-06 | 1910.1ms/step | 31.4 samples/s
|
| 292 |
+
Step 46790/50462 | Loss: 2.5119 | LR: 6.70e-06 | 1907.0ms/step | 31.5 samples/s
|
| 293 |
+
Step 46800/50462 | Loss: 2.4788 | LR: 6.66e-06 | 1905.5ms/step | 31.5 samples/s
|
| 294 |
+
Step 46810/50462 | Loss: 2.4766 | LR: 6.63e-06 | 1908.2ms/step | 31.4 samples/s
|
| 295 |
+
Step 46820/50462 | Loss: 2.4967 | LR: 6.59e-06 | 1909.3ms/step | 31.4 samples/s
|
| 296 |
+
Step 46830/50462 | Loss: 2.4762 | LR: 6.56e-06 | 1916.2ms/step | 31.3 samples/s
|
| 297 |
+
Step 46840/50462 | Loss: 2.4580 | LR: 6.52e-06 | 1914.4ms/step | 31.3 samples/s
|
| 298 |
+
Step 46850/50462 | Loss: 2.4431 | LR: 6.48e-06 | 1917.0ms/step | 31.3 samples/s
|
| 299 |
+
Step 46860/50462 | Loss: 2.4921 | LR: 6.45e-06 | 1916.7ms/step | 31.3 samples/s
|
| 300 |
+
Step 46870/50462 | Loss: 2.4721 | LR: 6.41e-06 | 1916.6ms/step | 31.3 samples/s
|
| 301 |
+
Step 46880/50462 | Loss: 2.4936 | LR: 6.38e-06 | 1917.5ms/step | 31.3 samples/s
|
| 302 |
+
Step 46890/50462 | Loss: 2.4726 | LR: 6.34e-06 | 1918.3ms/step | 31.3 samples/s
|
| 303 |
+
Step 46900/50462 | Loss: 2.4797 | LR: 6.31e-06 | 1911.5ms/step | 31.4 samples/s
|
| 304 |
+
Step 46910/50462 | Loss: 2.4617 | LR: 6.27e-06 | 1912.5ms/step | 31.4 samples/s
|
| 305 |
+
Step 46920/50462 | Loss: 2.4415 | LR: 6.24e-06 | 1910.7ms/step | 31.4 samples/s
|
| 306 |
+
Step 46930/50462 | Loss: 2.5101 | LR: 6.20e-06 | 1915.4ms/step | 31.3 samples/s
|
| 307 |
+
Step 46940/50462 | Loss: 2.4864 | LR: 6.17e-06 | 1916.3ms/step | 31.3 samples/s
|
| 308 |
+
Step 46950/50462 | Loss: 2.4699 | LR: 6.13e-06 | 1918.9ms/step | 31.3 samples/s
|
| 309 |
+
Step 46960/50462 | Loss: 2.4942 | LR: 6.10e-06 | 1919.5ms/step | 31.3 samples/s
|
| 310 |
+
Step 46970/50462 | Loss: 2.4690 | LR: 6.06e-06 | 1921.2ms/step | 31.2 samples/s
|
| 311 |
+
Step 46980/50462 | Loss: 2.5051 | LR: 6.03e-06 | 1917.1ms/step | 31.3 samples/s
|
| 312 |
+
Step 46990/50462 | Loss: 2.4799 | LR: 5.99e-06 | 1917.6ms/step | 31.3 samples/s
|
| 313 |
+
Step 47000/50462 | Loss: 2.4552 | LR: 5.96e-06 | 1920.4ms/step | 31.2 samples/s
|
| 314 |
+
Validation at step 47000: Loss = 2.4901, Perplexity = 12.06
|
| 315 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-47000
|
| 316 |
+
Step 47010/50462 | Loss: 2.4624 | LR: 5.93e-06 | 3341.0ms/step | 18.0 samples/s
|
| 317 |
+
Step 47020/50462 | Loss: 2.4509 | LR: 5.89e-06 | 1902.2ms/step | 31.5 samples/s
|
| 318 |
+
Step 47030/50462 | Loss: 2.4603 | LR: 5.86e-06 | 1908.6ms/step | 31.4 samples/s
|
| 319 |
+
Step 47040/50462 | Loss: 2.4880 | LR: 5.82e-06 | 1905.0ms/step | 31.5 samples/s
|
| 320 |
+
Step 47050/50462 | Loss: 2.4702 | LR: 5.79e-06 | 1905.8ms/step | 31.5 samples/s
|
| 321 |
+
Step 47060/50462 | Loss: 2.5039 | LR: 5.76e-06 | 1907.3ms/step | 31.5 samples/s
|
| 322 |
+
Step 47070/50462 | Loss: 2.4740 | LR: 5.72e-06 | 1909.7ms/step | 31.4 samples/s
|
| 323 |
+
Step 47080/50462 | Loss: 2.4761 | LR: 5.69e-06 | 1912.3ms/step | 31.4 samples/s
|
| 324 |
+
Step 47090/50462 | Loss: 2.4672 | LR: 5.65e-06 | 1912.0ms/step | 31.4 samples/s
|
| 325 |
+
Step 47100/50462 | Loss: 2.4649 | LR: 5.62e-06 | 1915.7ms/step | 31.3 samples/s
|
| 326 |
+
Step 47110/50462 | Loss: 2.4676 | LR: 5.59e-06 | 1916.0ms/step | 31.3 samples/s
|
| 327 |
+
Step 47120/50462 | Loss: 2.5185 | LR: 5.56e-06 | 1919.3ms/step | 31.3 samples/s
|
| 328 |
+
Step 47130/50462 | Loss: 2.4732 | LR: 5.52e-06 | 1909.7ms/step | 31.4 samples/s
|
| 329 |
+
Step 47140/50462 | Loss: 2.4938 | LR: 5.49e-06 | 1913.5ms/step | 31.4 samples/s
|
| 330 |
+
Step 47150/50462 | Loss: 2.4366 | LR: 5.46e-06 | 1916.0ms/step | 31.3 samples/s
|
| 331 |
+
Step 47160/50462 | Loss: 2.5063 | LR: 5.42e-06 | 1913.5ms/step | 31.4 samples/s
|
| 332 |
+
Step 47170/50462 | Loss: 2.4345 | LR: 5.39e-06 | 1914.0ms/step | 31.3 samples/s
|
| 333 |
+
Step 47180/50462 | Loss: 2.4708 | LR: 5.36e-06 | 1907.9ms/step | 31.4 samples/s
|
| 334 |
+
Step 47190/50462 | Loss: 2.4807 | LR: 5.33e-06 | 1907.9ms/step | 31.4 samples/s
|
| 335 |
+
Step 47200/50462 | Loss: 2.4695 | LR: 5.29e-06 | 1913.0ms/step | 31.4 samples/s
|
| 336 |
+
Step 47210/50462 | Loss: 2.4493 | LR: 5.26e-06 | 1916.7ms/step | 31.3 samples/s
|
| 337 |
+
Step 47220/50462 | Loss: 2.4852 | LR: 5.23e-06 | 1912.5ms/step | 31.4 samples/s
|
| 338 |
+
Step 47230/50462 | Loss: 2.4985 | LR: 5.20e-06 | 1906.4ms/step | 31.5 samples/s
|
| 339 |
+
Step 47240/50462 | Loss: 2.4811 | LR: 5.16e-06 | 1911.4ms/step | 31.4 samples/s
|
| 340 |
+
Step 47250/50462 | Loss: 2.4634 | LR: 5.13e-06 | 1913.6ms/step | 31.4 samples/s
|
| 341 |
+
Validation at step 47250: Loss = 2.4901, Perplexity = 12.06
|
| 342 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-47250
|
| 343 |
+
Step 47260/50462 | Loss: 2.4847 | LR: 5.10e-06 | 3343.7ms/step | 17.9 samples/s
|
| 344 |
+
Step 47270/50462 | Loss: 2.4549 | LR: 5.07e-06 | 1900.0ms/step | 31.6 samples/s
|
| 345 |
+
Step 47280/50462 | Loss: 2.4888 | LR: 5.04e-06 | 1907.6ms/step | 31.5 samples/s
|
| 346 |
+
Step 47290/50462 | Loss: 2.5044 | LR: 5.01e-06 | 1908.8ms/step | 31.4 samples/s
|
| 347 |
+
Step 47300/50462 | Loss: 2.4815 | LR: 4.97e-06 | 1911.3ms/step | 31.4 samples/s
|
| 348 |
+
Step 47310/50462 | Loss: 2.4447 | LR: 4.94e-06 | 1912.4ms/step | 31.4 samples/s
|
| 349 |
+
Step 47320/50462 | Loss: 2.5121 | LR: 4.91e-06 | 1914.7ms/step | 31.3 samples/s
|
| 350 |
+
Step 47330/50462 | Loss: 2.4989 | LR: 4.88e-06 | 1916.0ms/step | 31.3 samples/s
|
| 351 |
+
Step 47340/50462 | Loss: 2.5256 | LR: 4.85e-06 | 1911.9ms/step | 31.4 samples/s
|
| 352 |
+
Step 47350/50462 | Loss: 2.4920 | LR: 4.82e-06 | 1907.7ms/step | 31.5 samples/s
|
| 353 |
+
Step 47360/50462 | Loss: 2.4850 | LR: 4.79e-06 | 1915.6ms/step | 31.3 samples/s
|
| 354 |
+
Step 47370/50462 | Loss: 2.4795 | LR: 4.76e-06 | 1912.8ms/step | 31.4 samples/s
|
| 355 |
+
Step 47380/50462 | Loss: 2.5012 | LR: 4.73e-06 | 1914.7ms/step | 31.3 samples/s
|
| 356 |
+
Step 47390/50462 | Loss: 2.4646 | LR: 4.70e-06 | 1910.4ms/step | 31.4 samples/s
|
| 357 |
+
Step 47400/50462 | Loss: 2.4956 | LR: 4.67e-06 | 1908.9ms/step | 31.4 samples/s
|
| 358 |
+
Step 47410/50462 | Loss: 2.4658 | LR: 4.64e-06 | 1909.0ms/step | 31.4 samples/s
|
| 359 |
+
Step 47420/50462 | Loss: 2.4363 | LR: 4.61e-06 | 1910.8ms/step | 31.4 samples/s
|
| 360 |
+
Step 47430/50462 | Loss: 2.4672 | LR: 4.58e-06 | 1915.3ms/step | 31.3 samples/s
|
| 361 |
+
Step 47440/50462 | Loss: 2.4756 | LR: 4.55e-06 | 1915.0ms/step | 31.3 samples/s
|
| 362 |
+
Step 47450/50462 | Loss: 2.5001 | LR: 4.52e-06 | 1910.7ms/step | 31.4 samples/s
|
| 363 |
+
Step 47460/50462 | Loss: 2.4580 | LR: 4.49e-06 | 1914.3ms/step | 31.3 samples/s
|
| 364 |
+
Step 47470/50462 | Loss: 2.4965 | LR: 4.46e-06 | 1917.0ms/step | 31.3 samples/s
|
| 365 |
+
Step 47480/50462 | Loss: 2.4510 | LR: 4.43e-06 | 1913.8ms/step | 31.4 samples/s
|
| 366 |
+
Step 47490/50462 | Loss: 2.4520 | LR: 4.40e-06 | 1917.4ms/step | 31.3 samples/s
|
| 367 |
+
Step 47500/50462 | Loss: 2.4966 | LR: 4.37e-06 | 1917.7ms/step | 31.3 samples/s
|
| 368 |
+
Validation at step 47500: Loss = 2.4901, Perplexity = 12.06
|
| 369 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-47500
|
| 370 |
+
Step 47510/50462 | Loss: 2.4316 | LR: 4.34e-06 | 3368.5ms/step | 17.8 samples/s
|
| 371 |
+
Step 47520/50462 | Loss: 2.4603 | LR: 4.31e-06 | 1895.9ms/step | 31.6 samples/s
|
| 372 |
+
Step 47530/50462 | Loss: 2.4709 | LR: 4.28e-06 | 1903.4ms/step | 31.5 samples/s
|
| 373 |
+
Step 47540/50462 | Loss: 2.4910 | LR: 4.25e-06 | 1908.9ms/step | 31.4 samples/s
|
| 374 |
+
Step 47550/50462 | Loss: 2.5149 | LR: 4.22e-06 | 1910.1ms/step | 31.4 samples/s
|
| 375 |
+
Step 47560/50462 | Loss: 2.4452 | LR: 4.19e-06 | 1913.3ms/step | 31.4 samples/s
|
| 376 |
+
Step 47570/50462 | Loss: 2.4349 | LR: 4.16e-06 | 1913.5ms/step | 31.4 samples/s
|
| 377 |
+
Step 47580/50462 | Loss: 2.4714 | LR: 4.13e-06 | 1917.3ms/step | 31.3 samples/s
|
| 378 |
+
Step 47590/50462 | Loss: 2.4704 | LR: 4.11e-06 | 1917.9ms/step | 31.3 samples/s
|
| 379 |
+
Step 47600/50462 | Loss: 2.4717 | LR: 4.08e-06 | 1912.9ms/step | 31.4 samples/s
|
| 380 |
+
Step 47610/50462 | Loss: 2.4811 | LR: 4.05e-06 | 1915.2ms/step | 31.3 samples/s
|
| 381 |
+
Step 47620/50462 | Loss: 2.4519 | LR: 4.02e-06 | 1916.3ms/step | 31.3 samples/s
|
| 382 |
+
Step 47630/50462 | Loss: 2.5027 | LR: 3.99e-06 | 1915.4ms/step | 31.3 samples/s
|
| 383 |
+
Step 47640/50462 | Loss: 2.4838 | LR: 3.97e-06 | 1915.4ms/step | 31.3 samples/s
|
| 384 |
+
Step 47650/50462 | Loss: 2.4897 | LR: 3.94e-06 | 1915.4ms/step | 31.3 samples/s
|
| 385 |
+
Step 47660/50462 | Loss: 2.4577 | LR: 3.91e-06 | 1916.7ms/step | 31.3 samples/s
|
| 386 |
+
Step 47670/50462 | Loss: 2.4531 | LR: 3.88e-06 | 1915.3ms/step | 31.3 samples/s
|
| 387 |
+
Step 47680/50462 | Loss: 2.4537 | LR: 3.85e-06 | 1908.3ms/step | 31.4 samples/s
|
| 388 |
+
Step 47690/50462 | Loss: 2.4639 | LR: 3.83e-06 | 1912.7ms/step | 31.4 samples/s
|
| 389 |
+
Step 47700/50462 | Loss: 2.4432 | LR: 3.80e-06 | 1915.5ms/step | 31.3 samples/s
|
| 390 |
+
Step 47710/50462 | Loss: 2.4811 | LR: 3.77e-06 | 1915.3ms/step | 31.3 samples/s
|
| 391 |
+
Step 47720/50462 | Loss: 2.4361 | LR: 3.74e-06 | 1916.5ms/step | 31.3 samples/s
|
| 392 |
+
Step 47730/50462 | Loss: 2.4703 | LR: 3.72e-06 | 1913.9ms/step | 31.4 samples/s
|
| 393 |
+
Step 47740/50462 | Loss: 2.4726 | LR: 3.69e-06 | 1912.5ms/step | 31.4 samples/s
|
| 394 |
+
Step 47750/50462 | Loss: 2.4781 | LR: 3.66e-06 | 1909.8ms/step | 31.4 samples/s
|
| 395 |
+
Validation at step 47750: Loss = 2.4900, Perplexity = 12.06
|
| 396 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-47750
|
| 397 |
+
Step 47760/50462 | Loss: 2.5111 | LR: 3.64e-06 | 3287.8ms/step | 18.2 samples/s
|
| 398 |
+
Step 47770/50462 | Loss: 2.4886 | LR: 3.61e-06 | 1899.0ms/step | 31.6 samples/s
|
| 399 |
+
Step 47780/50462 | Loss: 2.4565 | LR: 3.58e-06 | 1900.5ms/step | 31.6 samples/s
|
| 400 |
+
Step 47790/50462 | Loss: 2.4379 | LR: 3.56e-06 | 1907.6ms/step | 31.5 samples/s
|
| 401 |
+
Step 47800/50462 | Loss: 2.4146 | LR: 3.53e-06 | 1905.6ms/step | 31.5 samples/s
|
| 402 |
+
Step 47810/50462 | Loss: 2.4706 | LR: 3.50e-06 | 1905.2ms/step | 31.5 samples/s
|
| 403 |
+
Step 47820/50462 | Loss: 2.5032 | LR: 3.48e-06 | 1905.3ms/step | 31.5 samples/s
|
| 404 |
+
Step 47830/50462 | Loss: 2.4674 | LR: 3.45e-06 | 1909.6ms/step | 31.4 samples/s
|
| 405 |
+
Step 47840/50462 | Loss: 2.4543 | LR: 3.42e-06 | 1902.9ms/step | 31.5 samples/s
|
| 406 |
+
Step 47850/50462 | Loss: 2.4133 | LR: 3.40e-06 | 1909.5ms/step | 31.4 samples/s
|
| 407 |
+
Step 47860/50462 | Loss: 2.4678 | LR: 3.37e-06 | 1913.8ms/step | 31.4 samples/s
|
| 408 |
+
Step 47870/50462 | Loss: 2.4520 | LR: 3.35e-06 | 1913.5ms/step | 31.4 samples/s
|
| 409 |
+
Step 47880/50462 | Loss: 2.4444 | LR: 3.32e-06 | 1911.4ms/step | 31.4 samples/s
|
| 410 |
+
Step 47890/50462 | Loss: 2.4562 | LR: 3.30e-06 | 1909.5ms/step | 31.4 samples/s
|
| 411 |
+
Step 47900/50462 | Loss: 2.5379 | LR: 3.27e-06 | 1904.5ms/step | 31.5 samples/s
|
| 412 |
+
Step 47910/50462 | Loss: 2.4666 | LR: 3.24e-06 | 1911.2ms/step | 31.4 samples/s
|
| 413 |
+
Step 47920/50462 | Loss: 2.4680 | LR: 3.22e-06 | 1910.5ms/step | 31.4 samples/s
|
| 414 |
+
Step 47930/50462 | Loss: 2.4551 | LR: 3.19e-06 | 1915.7ms/step | 31.3 samples/s
|
| 415 |
+
Step 47940/50462 | Loss: 2.4761 | LR: 3.17e-06 | 1917.3ms/step | 31.3 samples/s
|
| 416 |
+
Step 47950/50462 | Loss: 2.4609 | LR: 3.14e-06 | 1916.9ms/step | 31.3 samples/s
|
| 417 |
+
Step 47960/50462 | Loss: 2.4535 | LR: 3.12e-06 | 1909.8ms/step | 31.4 samples/s
|
| 418 |
+
Step 47970/50462 | Loss: 2.5055 | LR: 3.09e-06 | 1906.7ms/step | 31.5 samples/s
|
| 419 |
+
Step 47980/50462 | Loss: 2.4805 | LR: 3.07e-06 | 1908.0ms/step | 31.4 samples/s
|
| 420 |
+
Step 47990/50462 | Loss: 2.4892 | LR: 3.04e-06 | 1907.0ms/step | 31.5 samples/s
|
| 421 |
+
Step 48000/50462 | Loss: 2.4628 | LR: 3.02e-06 | 1908.8ms/step | 31.4 samples/s
|
| 422 |
+
Validation at step 48000: Loss = 2.4900, Perplexity = 12.06
|
| 423 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-48000
|
| 424 |
+
Step 48010/50462 | Loss: 2.5061 | LR: 3.00e-06 | 3305.5ms/step | 18.2 samples/s
|
| 425 |
+
Step 48020/50462 | Loss: 2.4775 | LR: 2.97e-06 | 1900.3ms/step | 31.6 samples/s
|
| 426 |
+
Step 48030/50462 | Loss: 2.4612 | LR: 2.95e-06 | 1906.7ms/step | 31.5 samples/s
|
| 427 |
+
Step 48040/50462 | Loss: 2.4721 | LR: 2.92e-06 | 1904.5ms/step | 31.5 samples/s
|
| 428 |
+
Step 48050/50462 | Loss: 2.4925 | LR: 2.90e-06 | 1903.6ms/step | 31.5 samples/s
|
| 429 |
+
Step 48060/50462 | Loss: 2.4860 | LR: 2.87e-06 | 1911.3ms/step | 31.4 samples/s
|
| 430 |
+
Step 48070/50462 | Loss: 2.5066 | LR: 2.85e-06 | 1902.2ms/step | 31.5 samples/s
|
| 431 |
+
Step 48080/50462 | Loss: 2.4686 | LR: 2.83e-06 | 1903.3ms/step | 31.5 samples/s
|
| 432 |
+
Step 48090/50462 | Loss: 2.4658 | LR: 2.80e-06 | 1906.8ms/step | 31.5 samples/s
|
| 433 |
+
Step 48100/50462 | Loss: 2.4794 | LR: 2.78e-06 | 1909.4ms/step | 31.4 samples/s
|
| 434 |
+
Step 48110/50462 | Loss: 2.4726 | LR: 2.76e-06 | 1908.9ms/step | 31.4 samples/s
|
| 435 |
+
Step 48120/50462 | Loss: 2.4690 | LR: 2.73e-06 | 1909.6ms/step | 31.4 samples/s
|
| 436 |
+
Step 48130/50462 | Loss: 2.4615 | LR: 2.71e-06 | 1912.3ms/step | 31.4 samples/s
|
| 437 |
+
Step 48140/50462 | Loss: 2.4621 | LR: 2.69e-06 | 1914.8ms/step | 31.3 samples/s
|
| 438 |
+
Step 48150/50462 | Loss: 2.4814 | LR: 2.66e-06 | 1914.1ms/step | 31.3 samples/s
|
| 439 |
+
Step 48160/50462 | Loss: 2.5106 | LR: 2.64e-06 | 1912.9ms/step | 31.4 samples/s
|
| 440 |
+
Step 48170/50462 | Loss: 2.4597 | LR: 2.62e-06 | 1914.4ms/step | 31.3 samples/s
|
| 441 |
+
Step 48180/50462 | Loss: 2.4684 | LR: 2.60e-06 | 1912.0ms/step | 31.4 samples/s
|
| 442 |
+
Step 48190/50462 | Loss: 2.4707 | LR: 2.57e-06 | 1902.7ms/step | 31.5 samples/s
|
| 443 |
+
Step 48200/50462 | Loss: 2.4805 | LR: 2.55e-06 | 1907.2ms/step | 31.5 samples/s
|
| 444 |
+
Step 48210/50462 | Loss: 2.4756 | LR: 2.53e-06 | 1912.5ms/step | 31.4 samples/s
|
| 445 |
+
Step 48220/50462 | Loss: 2.4593 | LR: 2.51e-06 | 1912.4ms/step | 31.4 samples/s
|
| 446 |
+
Step 48230/50462 | Loss: 2.4852 | LR: 2.48e-06 | 1908.8ms/step | 31.4 samples/s
|
| 447 |
+
Step 48240/50462 | Loss: 2.4623 | LR: 2.46e-06 | 1907.7ms/step | 31.5 samples/s
|
| 448 |
+
Step 48250/50462 | Loss: 2.4933 | LR: 2.44e-06 | 1908.7ms/step | 31.4 samples/s
|
| 449 |
+
Validation at step 48250: Loss = 2.4901, Perplexity = 12.06
|
| 450 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-48250
|
| 451 |
+
Step 48260/50462 | Loss: 2.4770 | LR: 2.42e-06 | 3310.0ms/step | 18.1 samples/s
|
| 452 |
+
Step 48270/50462 | Loss: 2.4649 | LR: 2.39e-06 | 1903.9ms/step | 31.5 samples/s
|
| 453 |
+
Step 48280/50462 | Loss: 2.4922 | LR: 2.37e-06 | 1909.7ms/step | 31.4 samples/s
|
| 454 |
+
Step 48290/50462 | Loss: 2.4995 | LR: 2.35e-06 | 1908.1ms/step | 31.4 samples/s
|
| 455 |
+
Step 48300/50462 | Loss: 2.4640 | LR: 2.33e-06 | 1909.8ms/step | 31.4 samples/s
|
| 456 |
+
Step 48310/50462 | Loss: 2.4677 | LR: 2.31e-06 | 1909.7ms/step | 31.4 samples/s
|
| 457 |
+
Step 48320/50462 | Loss: 2.4921 | LR: 2.29e-06 | 1909.8ms/step | 31.4 samples/s
|
| 458 |
+
Step 48330/50462 | Loss: 2.4470 | LR: 2.27e-06 | 1912.1ms/step | 31.4 samples/s
|
| 459 |
+
Step 48340/50462 | Loss: 2.4465 | LR: 2.24e-06 | 1910.1ms/step | 31.4 samples/s
|
| 460 |
+
Step 48350/50462 | Loss: 2.4001 | LR: 2.22e-06 | 1909.7ms/step | 31.4 samples/s
|
| 461 |
+
Step 48360/50462 | Loss: 2.5234 | LR: 2.20e-06 | 1912.4ms/step | 31.4 samples/s
|
| 462 |
+
Step 48370/50462 | Loss: 2.5115 | LR: 2.18e-06 | 1913.4ms/step | 31.4 samples/s
|
| 463 |
+
Step 48380/50462 | Loss: 2.4791 | LR: 2.16e-06 | 1915.2ms/step | 31.3 samples/s
|
| 464 |
+
Step 48390/50462 | Loss: 2.4327 | LR: 2.14e-06 | 1915.5ms/step | 31.3 samples/s
|
| 465 |
+
Step 48400/50462 | Loss: 2.4563 | LR: 2.12e-06 | 1911.0ms/step | 31.4 samples/s
|
| 466 |
+
Step 48410/50462 | Loss: 2.4858 | LR: 2.10e-06 | 1903.8ms/step | 31.5 samples/s
|
| 467 |
+
Step 48420/50462 | Loss: 2.4909 | LR: 2.08e-06 | 1906.7ms/step | 31.5 samples/s
|
| 468 |
+
Step 48430/50462 | Loss: 2.4595 | LR: 2.06e-06 | 1906.6ms/step | 31.5 samples/s
|
| 469 |
+
Step 48440/50462 | Loss: 2.4821 | LR: 2.04e-06 | 1907.6ms/step | 31.5 samples/s
|
| 470 |
+
Step 48450/50462 | Loss: 2.4367 | LR: 2.02e-06 | 1908.4ms/step | 31.4 samples/s
|
| 471 |
+
Step 48460/50462 | Loss: 2.4438 | LR: 2.00e-06 | 1910.2ms/step | 31.4 samples/s
|
| 472 |
+
Step 48470/50462 | Loss: 2.4829 | LR: 1.98e-06 | 1912.5ms/step | 31.4 samples/s
|
| 473 |
+
Step 48480/50462 | Loss: 2.4500 | LR: 1.96e-06 | 1913.3ms/step | 31.4 samples/s
|
| 474 |
+
Step 48490/50462 | Loss: 2.4617 | LR: 1.94e-06 | 1914.6ms/step | 31.3 samples/s
|
| 475 |
+
Step 48500/50462 | Loss: 2.4939 | LR: 1.92e-06 | 1907.6ms/step | 31.5 samples/s
|
| 476 |
+
Validation at step 48500: Loss = 2.4900, Perplexity = 12.06
|
| 477 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-48500
|
| 478 |
+
Step 48510/50462 | Loss: 2.4397 | LR: 1.90e-06 | 3285.8ms/step | 18.3 samples/s
|
| 479 |
+
Step 48520/50462 | Loss: 2.4789 | LR: 1.88e-06 | 1899.5ms/step | 31.6 samples/s
|
| 480 |
+
Step 48530/50462 | Loss: 2.5230 | LR: 1.86e-06 | 1905.6ms/step | 31.5 samples/s
|
| 481 |
+
Step 48540/50462 | Loss: 2.5059 | LR: 1.84e-06 | 1906.3ms/step | 31.5 samples/s
|
| 482 |
+
Step 48550/50462 | Loss: 2.4661 | LR: 1.82e-06 | 1907.5ms/step | 31.5 samples/s
|
| 483 |
+
Step 48560/50462 | Loss: 2.4521 | LR: 1.80e-06 | 1907.5ms/step | 31.5 samples/s
|
| 484 |
+
Step 48570/50462 | Loss: 2.4919 | LR: 1.78e-06 | 1908.7ms/step | 31.4 samples/s
|
| 485 |
+
Step 48580/50462 | Loss: 2.4468 | LR: 1.77e-06 | 1914.7ms/step | 31.3 samples/s
|
| 486 |
+
Step 48590/50462 | Loss: 2.4751 | LR: 1.75e-06 | 1910.5ms/step | 31.4 samples/s
|
| 487 |
+
Step 48600/50462 | Loss: 2.4989 | LR: 1.73e-06 | 1909.4ms/step | 31.4 samples/s
|
| 488 |
+
Step 48610/50462 | Loss: 2.4311 | LR: 1.71e-06 | 1909.3ms/step | 31.4 samples/s
|
| 489 |
+
Step 48620/50462 | Loss: 2.4916 | LR: 1.69e-06 | 1909.1ms/step | 31.4 samples/s
|
| 490 |
+
Step 48630/50462 | Loss: 2.5141 | LR: 1.67e-06 | 1908.9ms/step | 31.4 samples/s
|
| 491 |
+
Step 48640/50462 | Loss: 2.4835 | LR: 1.66e-06 | 1910.3ms/step | 31.4 samples/s
|
| 492 |
+
Step 48650/50462 | Loss: 2.4688 | LR: 1.64e-06 | 1912.3ms/step | 31.4 samples/s
|
| 493 |
+
Step 48660/50462 | Loss: 2.4443 | LR: 1.62e-06 | 1915.3ms/step | 31.3 samples/s
|
| 494 |
+
Step 48670/50462 | Loss: 2.4824 | LR: 1.60e-06 | 1915.3ms/step | 31.3 samples/s
|
| 495 |
+
Step 48680/50462 | Loss: 2.4931 | LR: 1.58e-06 | 1915.0ms/step | 31.3 samples/s
|
| 496 |
+
Step 48690/50462 | Loss: 2.4500 | LR: 1.57e-06 | 1913.5ms/step | 31.4 samples/s
|
| 497 |
+
Step 48700/50462 | Loss: 2.4946 | LR: 1.55e-06 | 1911.0ms/step | 31.4 samples/s
|
| 498 |
+
Step 48710/50462 | Loss: 2.5140 | LR: 1.53e-06 | 1912.4ms/step | 31.4 samples/s
|
| 499 |
+
Step 48720/50462 | Loss: 2.4698 | LR: 1.51e-06 | 1907.2ms/step | 31.5 samples/s
|
| 500 |
+
Step 48730/50462 | Loss: 2.4668 | LR: 1.50e-06 | 1907.9ms/step | 31.4 samples/s
|
| 501 |
+
Step 48740/50462 | Loss: 2.4944 | LR: 1.48e-06 | 1909.4ms/step | 31.4 samples/s
|
| 502 |
+
Step 48750/50462 | Loss: 2.4483 | LR: 1.46e-06 | 1904.7ms/step | 31.5 samples/s
|
| 503 |
+
Validation at step 48750: Loss = 2.4900, Perplexity = 12.06
|
| 504 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-48750
|
| 505 |
+
Step 48760/50462 | Loss: 2.4584 | LR: 1.44e-06 | 3303.3ms/step | 18.2 samples/s
|
| 506 |
+
Step 48770/50462 | Loss: 2.5226 | LR: 1.43e-06 | 1903.9ms/step | 31.5 samples/s
|
| 507 |
+
Step 48780/50462 | Loss: 2.4615 | LR: 1.41e-06 | 1909.7ms/step | 31.4 samples/s
|
| 508 |
+
Step 48790/50462 | Loss: 2.5035 | LR: 1.39e-06 | 1912.4ms/step | 31.4 samples/s
|
| 509 |
+
Step 48800/50462 | Loss: 2.4687 | LR: 1.38e-06 | 1912.7ms/step | 31.4 samples/s
|
| 510 |
+
Step 48810/50462 | Loss: 2.4879 | LR: 1.36e-06 | 1907.6ms/step | 31.5 samples/s
|
| 511 |
+
Step 48820/50462 | Loss: 2.4976 | LR: 1.34e-06 | 1909.0ms/step | 31.4 samples/s
|
| 512 |
+
Step 48830/50462 | Loss: 2.4699 | LR: 1.33e-06 | 1911.0ms/step | 31.4 samples/s
|
| 513 |
+
Step 48840/50462 | Loss: 2.4762 | LR: 1.31e-06 | 1906.6ms/step | 31.5 samples/s
|
| 514 |
+
Step 48850/50462 | Loss: 2.5188 | LR: 1.30e-06 | 1902.6ms/step | 31.5 samples/s
|
| 515 |
+
Step 48860/50462 | Loss: 2.4864 | LR: 1.28e-06 | 1906.1ms/step | 31.5 samples/s
|
| 516 |
+
Step 48870/50462 | Loss: 2.5057 | LR: 1.26e-06 | 1909.1ms/step | 31.4 samples/s
|
| 517 |
+
Step 48880/50462 | Loss: 2.5013 | LR: 1.25e-06 | 1911.8ms/step | 31.4 samples/s
|
| 518 |
+
Step 48890/50462 | Loss: 2.4523 | LR: 1.23e-06 | 1912.4ms/step | 31.4 samples/s
|
| 519 |
+
Step 48900/50462 | Loss: 2.4712 | LR: 1.22e-06 | 1912.3ms/step | 31.4 samples/s
|
| 520 |
+
Step 48910/50462 | Loss: 2.4895 | LR: 1.20e-06 | 1912.0ms/step | 31.4 samples/s
|
| 521 |
+
Step 48920/50462 | Loss: 2.4680 | LR: 1.19e-06 | 1908.9ms/step | 31.4 samples/s
|
| 522 |
+
Step 48930/50462 | Loss: 2.4748 | LR: 1.17e-06 | 1906.5ms/step | 31.5 samples/s
|
| 523 |
+
Step 48940/50462 | Loss: 2.4856 | LR: 1.16e-06 | 1904.8ms/step | 31.5 samples/s
|
| 524 |
+
Step 48950/50462 | Loss: 2.4704 | LR: 1.14e-06 | 1910.9ms/step | 31.4 samples/s
|
| 525 |
+
Step 48960/50462 | Loss: 2.4548 | LR: 1.13e-06 | 1914.1ms/step | 31.3 samples/s
|
| 526 |
+
Step 48970/50462 | Loss: 2.5006 | LR: 1.11e-06 | 1908.5ms/step | 31.4 samples/s
|
| 527 |
+
Step 48980/50462 | Loss: 2.4785 | LR: 1.10e-06 | 1908.0ms/step | 31.4 samples/s
|
| 528 |
+
Step 48990/50462 | Loss: 2.5322 | LR: 1.08e-06 | 1905.5ms/step | 31.5 samples/s
|
| 529 |
+
Step 49000/50462 | Loss: 2.4630 | LR: 1.07e-06 | 1910.3ms/step | 31.4 samples/s
|
| 530 |
+
Validation at step 49000: Loss = 2.4901, Perplexity = 12.06
|
| 531 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-49000
|
| 532 |
+
Step 49010/50462 | Loss: 2.4895 | LR: 1.05e-06 | 3311.8ms/step | 18.1 samples/s
|
| 533 |
+
Step 49020/50462 | Loss: 2.4577 | LR: 1.04e-06 | 1900.6ms/step | 31.6 samples/s
|
| 534 |
+
Step 49030/50462 | Loss: 2.5162 | LR: 1.02e-06 | 1903.8ms/step | 31.5 samples/s
|
| 535 |
+
Step 49040/50462 | Loss: 2.4692 | LR: 1.01e-06 | 1899.1ms/step | 31.6 samples/s
|
| 536 |
+
Step 49050/50462 | Loss: 2.4755 | LR: 9.95e-07 | 1903.7ms/step | 31.5 samples/s
|
| 537 |
+
Step 49060/50462 | Loss: 2.5002 | LR: 9.81e-07 | 1907.8ms/step | 31.5 samples/s
|
| 538 |
+
Step 49070/50462 | Loss: 2.4834 | LR: 9.67e-07 | 1910.1ms/step | 31.4 samples/s
|
| 539 |
+
Step 49080/50462 | Loss: 2.4975 | LR: 9.53e-07 | 1915.7ms/step | 31.3 samples/s
|
| 540 |
+
Step 49090/50462 | Loss: 2.4828 | LR: 9.39e-07 | 1915.9ms/step | 31.3 samples/s
|
| 541 |
+
Step 49100/50462 | Loss: 2.4557 | LR: 9.25e-07 | 1913.9ms/step | 31.3 samples/s
|
| 542 |
+
Step 49110/50462 | Loss: 2.4651 | LR: 9.12e-07 | 1917.4ms/step | 31.3 samples/s
|
| 543 |
+
Step 49120/50462 | Loss: 2.4577 | LR: 8.99e-07 | 1916.2ms/step | 31.3 samples/s
|
| 544 |
+
Step 49130/50462 | Loss: 2.4602 | LR: 8.85e-07 | 1918.6ms/step | 31.3 samples/s
|
| 545 |
+
Step 49140/50462 | Loss: 2.4908 | LR: 8.72e-07 | 1915.9ms/step | 31.3 samples/s
|
| 546 |
+
Step 49150/50462 | Loss: 2.4992 | LR: 8.59e-07 | 1918.3ms/step | 31.3 samples/s
|
| 547 |
+
Step 49160/50462 | Loss: 2.4805 | LR: 8.46e-07 | 1916.7ms/step | 31.3 samples/s
|
| 548 |
+
Step 49170/50462 | Loss: 2.4699 | LR: 8.33e-07 | 1918.0ms/step | 31.3 samples/s
|
| 549 |
+
Step 49180/50462 | Loss: 2.4697 | LR: 8.20e-07 | 1917.8ms/step | 31.3 samples/s
|
| 550 |
+
Step 49190/50462 | Loss: 2.4488 | LR: 8.07e-07 | 1913.9ms/step | 31.3 samples/s
|
| 551 |
+
Step 49200/50462 | Loss: 2.4835 | LR: 7.95e-07 | 1917.1ms/step | 31.3 samples/s
|
| 552 |
+
Step 49210/50462 | Loss: 2.4457 | LR: 7.82e-07 | 1916.6ms/step | 31.3 samples/s
|
| 553 |
+
Step 49220/50462 | Loss: 2.4478 | LR: 7.70e-07 | 1912.9ms/step | 31.4 samples/s
|
| 554 |
+
Step 49230/50462 | Loss: 2.4890 | LR: 7.57e-07 | 1913.5ms/step | 31.4 samples/s
|
| 555 |
+
Step 49240/50462 | Loss: 2.4466 | LR: 7.45e-07 | 1909.5ms/step | 31.4 samples/s
|
| 556 |
+
Step 49250/50462 | Loss: 2.4853 | LR: 7.33e-07 | 1908.4ms/step | 31.4 samples/s
|
| 557 |
+
Validation at step 49250: Loss = 2.4900, Perplexity = 12.06
|
| 558 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-49250
|
| 559 |
+
Step 49260/50462 | Loss: 2.4478 | LR: 7.21e-07 | 3327.1ms/step | 18.0 samples/s
|
| 560 |
+
Step 49270/50462 | Loss: 2.4730 | LR: 7.09e-07 | 1897.3ms/step | 31.6 samples/s
|
| 561 |
+
Step 49280/50462 | Loss: 2.5152 | LR: 6.97e-07 | 1901.8ms/step | 31.5 samples/s
|
| 562 |
+
Step 49290/50462 | Loss: 2.4851 | LR: 6.85e-07 | 1903.6ms/step | 31.5 samples/s
|
| 563 |
+
Step 49300/50462 | Loss: 2.4753 | LR: 6.74e-07 | 1898.5ms/step | 31.6 samples/s
|
| 564 |
+
Step 49310/50462 | Loss: 2.4799 | LR: 6.62e-07 | 1907.3ms/step | 31.5 samples/s
|
| 565 |
+
Step 49320/50462 | Loss: 2.4474 | LR: 6.51e-07 | 1909.5ms/step | 31.4 samples/s
|
| 566 |
+
Step 49330/50462 | Loss: 2.4250 | LR: 6.39e-07 | 1912.0ms/step | 31.4 samples/s
|
| 567 |
+
Step 49340/50462 | Loss: 2.4870 | LR: 6.28e-07 | 1904.4ms/step | 31.5 samples/s
|
| 568 |
+
Step 49350/50462 | Loss: 2.4754 | LR: 6.17e-07 | 1898.4ms/step | 31.6 samples/s
|
| 569 |
+
Step 49360/50462 | Loss: 2.5245 | LR: 6.06e-07 | 1905.4ms/step | 31.5 samples/s
|
| 570 |
+
Step 49370/50462 | Loss: 2.4859 | LR: 5.95e-07 | 1911.0ms/step | 31.4 samples/s
|
| 571 |
+
Step 49380/50462 | Loss: 2.5336 | LR: 5.84e-07 | 1911.8ms/step | 31.4 samples/s
|
| 572 |
+
Step 49390/50462 | Loss: 2.4835 | LR: 5.73e-07 | 1904.7ms/step | 31.5 samples/s
|
| 573 |
+
Step 49400/50462 | Loss: 2.4531 | LR: 5.63e-07 | 1906.3ms/step | 31.5 samples/s
|
| 574 |
+
Step 49410/50462 | Loss: 2.4668 | LR: 5.52e-07 | 1908.1ms/step | 31.4 samples/s
|
| 575 |
+
Step 49420/50462 | Loss: 2.5116 | LR: 5.42e-07 | 1911.8ms/step | 31.4 samples/s
|
| 576 |
+
Step 49430/50462 | Loss: 2.4199 | LR: 5.31e-07 | 1907.2ms/step | 31.5 samples/s
|
| 577 |
+
Step 49440/50462 | Loss: 2.4789 | LR: 5.21e-07 | 1909.2ms/step | 31.4 samples/s
|
| 578 |
+
Step 49450/50462 | Loss: 2.4706 | LR: 5.11e-07 | 1910.9ms/step | 31.4 samples/s
|
| 579 |
+
Step 49460/50462 | Loss: 2.4554 | LR: 5.01e-07 | 1913.5ms/step | 31.4 samples/s
|
| 580 |
+
Step 49470/50462 | Loss: 2.4686 | LR: 4.91e-07 | 1914.3ms/step | 31.3 samples/s
|
| 581 |
+
Step 49480/50462 | Loss: 2.5266 | LR: 4.81e-07 | 1908.4ms/step | 31.4 samples/s
|
| 582 |
+
Step 49490/50462 | Loss: 2.4479 | LR: 4.72e-07 | 1900.2ms/step | 31.6 samples/s
|
| 583 |
+
Step 49500/50462 | Loss: 2.4903 | LR: 4.62e-07 | 1904.9ms/step | 31.5 samples/s
|
| 584 |
+
Validation at step 49500: Loss = 2.4900, Perplexity = 12.06
|
| 585 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-49500
|
| 586 |
+
Step 49510/50462 | Loss: 2.4860 | LR: 4.52e-07 | 3325.1ms/step | 18.0 samples/s
|
| 587 |
+
Step 49520/50462 | Loss: 2.5024 | LR: 4.43e-07 | 1896.0ms/step | 31.6 samples/s
|
| 588 |
+
Step 49530/50462 | Loss: 2.4832 | LR: 4.34e-07 | 1902.1ms/step | 31.5 samples/s
|
| 589 |
+
Step 49540/50462 | Loss: 2.4791 | LR: 4.24e-07 | 1908.0ms/step | 31.4 samples/s
|
| 590 |
+
Step 49550/50462 | Loss: 2.4976 | LR: 4.15e-07 | 1912.1ms/step | 31.4 samples/s
|
| 591 |
+
Step 49560/50462 | Loss: 2.4936 | LR: 4.06e-07 | 1905.1ms/step | 31.5 samples/s
|
| 592 |
+
Step 49570/50462 | Loss: 2.5317 | LR: 3.97e-07 | 1903.5ms/step | 31.5 samples/s
|
| 593 |
+
Step 49580/50462 | Loss: 2.4921 | LR: 3.88e-07 | 1901.0ms/step | 31.6 samples/s
|
| 594 |
+
Step 49590/50462 | Loss: 2.4729 | LR: 3.79e-07 | 1898.4ms/step | 31.6 samples/s
|
| 595 |
+
Step 49600/50462 | Loss: 2.4589 | LR: 3.71e-07 | 1901.5ms/step | 31.6 samples/s
|
| 596 |
+
Step 49610/50462 | Loss: 2.4676 | LR: 3.62e-07 | 1906.2ms/step | 31.5 samples/s
|
| 597 |
+
Step 49620/50462 | Loss: 2.4817 | LR: 3.54e-07 | 1906.7ms/step | 31.5 samples/s
|
| 598 |
+
Step 49630/50462 | Loss: 2.4584 | LR: 3.45e-07 | 1905.5ms/step | 31.5 samples/s
|
| 599 |
+
Step 49640/50462 | Loss: 2.4275 | LR: 3.37e-07 | 1905.5ms/step | 31.5 samples/s
|
| 600 |
+
Step 49650/50462 | Loss: 2.5198 | LR: 3.29e-07 | 1905.8ms/step | 31.5 samples/s
|
| 601 |
+
Step 49660/50462 | Loss: 2.4816 | LR: 3.21e-07 | 1910.9ms/step | 31.4 samples/s
|
| 602 |
+
Step 49670/50462 | Loss: 2.5159 | LR: 3.13e-07 | 1913.2ms/step | 31.4 samples/s
|
| 603 |
+
Step 49680/50462 | Loss: 2.4862 | LR: 3.05e-07 | 1904.9ms/step | 31.5 samples/s
|
| 604 |
+
Step 49690/50462 | Loss: 2.4551 | LR: 2.97e-07 | 1907.3ms/step | 31.5 samples/s
|
| 605 |
+
Step 49700/50462 | Loss: 2.4532 | LR: 2.90e-07 | 1910.3ms/step | 31.4 samples/s
|
| 606 |
+
Step 49710/50462 | Loss: 2.4600 | LR: 2.82e-07 | 1912.2ms/step | 31.4 samples/s
|
| 607 |
+
Step 49720/50462 | Loss: 2.4669 | LR: 2.75e-07 | 1906.1ms/step | 31.5 samples/s
|
| 608 |
+
Step 49730/50462 | Loss: 2.4915 | LR: 2.67e-07 | 1908.6ms/step | 31.4 samples/s
|
| 609 |
+
Step 49740/50462 | Loss: 2.5192 | LR: 2.60e-07 | 1913.2ms/step | 31.4 samples/s
|
| 610 |
+
Step 49750/50462 | Loss: 2.4817 | LR: 2.53e-07 | 1916.1ms/step | 31.3 samples/s
|
| 611 |
+
Validation at step 49750: Loss = 2.4900, Perplexity = 12.06
|
| 612 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-49750
|
| 613 |
+
Step 49760/50462 | Loss: 2.4750 | LR: 2.46e-07 | 3302.3ms/step | 18.2 samples/s
|
| 614 |
+
Step 49770/50462 | Loss: 2.4494 | LR: 2.39e-07 | 1894.2ms/step | 31.7 samples/s
|
| 615 |
+
Step 49780/50462 | Loss: 2.4265 | LR: 2.32e-07 | 1901.3ms/step | 31.6 samples/s
|
| 616 |
+
Step 49790/50462 | Loss: 2.4522 | LR: 2.25e-07 | 1905.4ms/step | 31.5 samples/s
|
| 617 |
+
Step 49800/50462 | Loss: 2.4807 | LR: 2.19e-07 | 1906.7ms/step | 31.5 samples/s
|
| 618 |
+
Step 49810/50462 | Loss: 2.4546 | LR: 2.12e-07 | 1907.5ms/step | 31.5 samples/s
|
| 619 |
+
Step 49820/50462 | Loss: 2.4555 | LR: 2.06e-07 | 1910.1ms/step | 31.4 samples/s
|
| 620 |
+
Step 49830/50462 | Loss: 2.4801 | LR: 1.99e-07 | 1912.0ms/step | 31.4 samples/s
|
| 621 |
+
Step 49840/50462 | Loss: 2.5217 | LR: 1.93e-07 | 1915.9ms/step | 31.3 samples/s
|
| 622 |
+
Step 49850/50462 | Loss: 2.5176 | LR: 1.87e-07 | 1911.7ms/step | 31.4 samples/s
|
| 623 |
+
Step 49860/50462 | Loss: 2.4887 | LR: 1.81e-07 | 1912.8ms/step | 31.4 samples/s
|
| 624 |
+
Step 49870/50462 | Loss: 2.4746 | LR: 1.75e-07 | 1906.6ms/step | 31.5 samples/s
|
| 625 |
+
Step 49880/50462 | Loss: 2.4798 | LR: 1.69e-07 | 1906.0ms/step | 31.5 samples/s
|
| 626 |
+
Step 49890/50462 | Loss: 2.4532 | LR: 1.63e-07 | 1900.5ms/step | 31.6 samples/s
|
| 627 |
+
Step 49900/50462 | Loss: 2.4620 | LR: 1.58e-07 | 1904.7ms/step | 31.5 samples/s
|
| 628 |
+
Step 49910/50462 | Loss: 2.5089 | LR: 1.52e-07 | 1911.8ms/step | 31.4 samples/s
|
| 629 |
+
Step 49920/50462 | Loss: 2.4981 | LR: 1.47e-07 | 1911.1ms/step | 31.4 samples/s
|
| 630 |
+
Step 49930/50462 | Loss: 2.4900 | LR: 1.41e-07 | 1912.9ms/step | 31.4 samples/s
|
| 631 |
+
Step 49940/50462 | Loss: 2.4706 | LR: 1.36e-07 | 1912.5ms/step | 31.4 samples/s
|
| 632 |
+
Step 49950/50462 | Loss: 2.4545 | LR: 1.31e-07 | 1908.8ms/step | 31.4 samples/s
|
| 633 |
+
Step 49960/50462 | Loss: 2.4589 | LR: 1.26e-07 | 1909.6ms/step | 31.4 samples/s
|
| 634 |
+
Step 49970/50462 | Loss: 2.4720 | LR: 1.21e-07 | 1906.3ms/step | 31.5 samples/s
|
| 635 |
+
Step 49980/50462 | Loss: 2.4616 | LR: 1.16e-07 | 1907.5ms/step | 31.5 samples/s
|
| 636 |
+
Step 49990/50462 | Loss: 2.4920 | LR: 1.11e-07 | 1905.8ms/step | 31.5 samples/s
|
| 637 |
+
Step 50000/50462 | Loss: 2.4783 | LR: 1.07e-07 | 1904.9ms/step | 31.5 samples/s
|
| 638 |
+
Validation at step 50000: Loss = 2.4900, Perplexity = 12.06
|
| 639 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-50000
|
| 640 |
+
Step 50010/50462 | Loss: 2.4928 | LR: 1.02e-07 | 3391.7ms/step | 17.7 samples/s
|
| 641 |
+
Step 50020/50462 | Loss: 2.4954 | LR: 9.75e-08 | 1900.1ms/step | 31.6 samples/s
|
| 642 |
+
Step 50030/50462 | Loss: 2.5153 | LR: 9.32e-08 | 1904.9ms/step | 31.5 samples/s
|
| 643 |
+
Step 50040/50462 | Loss: 2.4806 | LR: 8.89e-08 | 1904.1ms/step | 31.5 samples/s
|
| 644 |
+
Step 50050/50462 | Loss: 2.5078 | LR: 8.47e-08 | 1913.3ms/step | 31.4 samples/s
|
| 645 |
+
Step 50060/50462 | Loss: 2.4572 | LR: 8.07e-08 | 1913.1ms/step | 31.4 samples/s
|
| 646 |
+
Step 50070/50462 | Loss: 2.4933 | LR: 7.67e-08 | 1908.5ms/step | 31.4 samples/s
|
| 647 |
+
Step 50080/50462 | Loss: 2.4554 | LR: 7.28e-08 | 1910.4ms/step | 31.4 samples/s
|
| 648 |
+
Step 50090/50462 | Loss: 2.4424 | LR: 6.91e-08 | 1909.8ms/step | 31.4 samples/s
|
| 649 |
+
Step 50100/50462 | Loss: 2.5165 | LR: 6.54e-08 | 1913.4ms/step | 31.4 samples/s
|
| 650 |
+
Step 50110/50462 | Loss: 2.4568 | LR: 6.19e-08 | 1913.5ms/step | 31.4 samples/s
|
| 651 |
+
Step 50120/50462 | Loss: 2.5113 | LR: 5.84e-08 | 1905.6ms/step | 31.5 samples/s
|
| 652 |
+
Step 50130/50462 | Loss: 2.4661 | LR: 5.50e-08 | 1907.2ms/step | 31.5 samples/s
|
| 653 |
+
Step 50140/50462 | Loss: 2.4560 | LR: 5.18e-08 | 1907.8ms/step | 31.4 samples/s
|
| 654 |
+
Step 50150/50462 | Loss: 2.4569 | LR: 4.86e-08 | 1912.6ms/step | 31.4 samples/s
|
| 655 |
+
Step 50160/50462 | Loss: 2.4925 | LR: 4.55e-08 | 1916.7ms/step | 31.3 samples/s
|
| 656 |
+
Step 50170/50462 | Loss: 2.4681 | LR: 4.26e-08 | 1910.4ms/step | 31.4 samples/s
|
| 657 |
+
Step 50180/50462 | Loss: 2.4736 | LR: 3.97e-08 | 1910.2ms/step | 31.4 samples/s
|
| 658 |
+
Step 50190/50462 | Loss: 2.4314 | LR: 3.69e-08 | 1906.8ms/step | 31.5 samples/s
|
| 659 |
+
Step 50200/50462 | Loss: 2.4717 | LR: 3.43e-08 | 1910.2ms/step | 31.4 samples/s
|
| 660 |
+
Step 50210/50462 | Loss: 2.4878 | LR: 3.17e-08 | 1910.3ms/step | 31.4 samples/s
|
| 661 |
+
Step 50220/50462 | Loss: 2.4418 | LR: 2.92e-08 | 1907.9ms/step | 31.4 samples/s
|
| 662 |
+
Step 50230/50462 | Loss: 2.4675 | LR: 2.69e-08 | 1908.0ms/step | 31.4 samples/s
|
| 663 |
+
Step 50240/50462 | Loss: 2.4533 | LR: 2.46e-08 | 1909.5ms/step | 31.4 samples/s
|
| 664 |
+
Step 50250/50462 | Loss: 2.5150 | LR: 2.24e-08 | 1911.7ms/step | 31.4 samples/s
|
| 665 |
+
Validation at step 50250: Loss = 2.4900, Perplexity = 12.06
|
| 666 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-50250
|
| 667 |
+
Step 50260/50462 | Loss: 2.4811 | LR: 2.04e-08 | 3287.0ms/step | 18.3 samples/s
|
| 668 |
+
Step 50270/50462 | Loss: 2.5083 | LR: 1.84e-08 | 1897.4ms/step | 31.6 samples/s
|
| 669 |
+
Step 50280/50462 | Loss: 2.5510 | LR: 1.65e-08 | 1899.6ms/step | 31.6 samples/s
|
| 670 |
+
Step 50290/50462 | Loss: 2.5288 | LR: 1.48e-08 | 1906.2ms/step | 31.5 samples/s
|
| 671 |
+
Step 50300/50462 | Loss: 2.4518 | LR: 1.31e-08 | 1906.4ms/step | 31.5 samples/s
|
| 672 |
+
Step 50310/50462 | Loss: 2.5093 | LR: 1.15e-08 | 1903.1ms/step | 31.5 samples/s
|
| 673 |
+
Step 50320/50462 | Loss: 2.5140 | LR: 1.01e-08 | 1909.6ms/step | 31.4 samples/s
|
| 674 |
+
Step 50330/50462 | Loss: 2.4529 | LR: 8.70e-09 | 1909.9ms/step | 31.4 samples/s
|
| 675 |
+
Step 50340/50462 | Loss: 2.4961 | LR: 7.43e-09 | 1909.6ms/step | 31.4 samples/s
|
| 676 |
+
Step 50350/50462 | Loss: 2.4958 | LR: 6.26e-09 | 1903.3ms/step | 31.5 samples/s
|
| 677 |
+
Step 50360/50462 | Loss: 2.4846 | LR: 5.19e-09 | 1909.6ms/step | 31.4 samples/s
|
| 678 |
+
Step 50370/50462 | Loss: 2.4880 | LR: 4.23e-09 | 1911.7ms/step | 31.4 samples/s
|
| 679 |
+
Step 50380/50462 | Loss: 2.5012 | LR: 3.36e-09 | 1915.7ms/step | 31.3 samples/s
|
| 680 |
+
Step 50390/50462 | Loss: 2.5293 | LR: 2.59e-09 | 1914.1ms/step | 31.3 samples/s
|
| 681 |
+
Step 50400/50462 | Loss: 2.4942 | LR: 1.92e-09 | 1915.9ms/step | 31.3 samples/s
|
| 682 |
+
Step 50410/50462 | Loss: 2.4957 | LR: 1.35e-09 | 1906.4ms/step | 31.5 samples/s
|
| 683 |
+
Step 50420/50462 | Loss: 2.4659 | LR: 8.81e-10 | 1910.2ms/step | 31.4 samples/s
|
| 684 |
+
Step 50430/50462 | Loss: 2.4412 | LR: 5.11e-10 | 1914.8ms/step | 31.3 samples/s
|
| 685 |
+
Step 50440/50462 | Loss: 2.4737 | LR: 2.42e-10 | 1913.6ms/step | 31.4 samples/s
|
| 686 |
+
Step 50450/50462 | Loss: 2.4546 | LR: 7.19e-11 | 1915.0ms/step | 31.3 samples/s
|
| 687 |
+
Step 50460/50462 | Loss: 2.4949 | LR: 2.00e-12 | 1913.7ms/step | 31.4 samples/s
|
| 688 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_dense_seed71/run_20260823_140253/checkpoint-50462
|
| 689 |
+
wandb: updating run metadata
|
| 690 |
+
wandb: uploading output.log; uploading wandb-summary.json; uploading config.yaml
|
| 691 |
+
wandb: uploading history steps 5045-5045, summary, console lines 490-490
|
| 692 |
+
wandb:
|
| 693 |
+
wandb: Run history:
|
| 694 |
+
wandb: eval/dense_interval_active ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 695 |
+
wandb: eval/dense_interval_end ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 696 |
+
wandb: eval/dense_interval_start ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 697 |
+
wandb: eval/loss ▇██▄▄▇▃▃▄▃▃▆▁▃▁▃▃
|
| 698 |
+
wandb: eval/perplexity ▇██▄▄▇▃▃▄▃▃▆▁▃▁▃▃
|
| 699 |
+
wandb: eval/uses_dense_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 700 |
+
wandb: eval/uses_sparse_attention ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
| 701 |
+
wandb: speed/ms_per_step ▁▁█▁▁▁▁▇▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▇▁▁▁▁▁▁▁▁▁▁▁▁
|
| 702 |
+
wandb: speed/samples_per_sec ███▁█████████▁██████████████████████████
|
| 703 |
+
wandb: speed/steps_per_sec ████████▁███████████████▁█████████▁█████
|
| 704 |
+
wandb: +4 ...
|
| 705 |
+
wandb:
|
| 706 |
+
wandb: Run summary:
|
| 707 |
+
wandb: eval/dense_interval_active 0
|
| 708 |
+
wandb: eval/dense_interval_end -1
|
| 709 |
+
wandb: eval/dense_interval_start -1
|
| 710 |
+
wandb: eval/loss 2.49004
|
| 711 |
+
wandb: eval/perplexity 12.06174
|
| 712 |
+
wandb: eval/uses_dense_attention 1
|
| 713 |
+
wandb: eval/uses_sparse_attention 0
|
| 714 |
+
wandb: speed/ms_per_step 1913.73353
|
| 715 |
+
wandb: speed/samples_per_sec 31.35233
|
| 716 |
+
wandb: speed/steps_per_sec 0.52254
|
| 717 |
+
wandb: +4 ...
|
| 718 |
+
wandb:
|
| 719 |
+
wandb: 🚀 View run 1.2b dense 8k seed71 6gpu at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/qctui00g
|
| 720 |
+
wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 721 |
+
wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
|
| 722 |
+
wandb: Find logs at: ./wandb/run-20260826_125030-qctui00g/logs
|
| 723 |
+
Training complete!
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_2gpu.log
ADDED
|
@@ -0,0 +1,543 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107",
|
| 16 |
+
"resume_from_checkpoint": null,
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 2,
|
| 21 |
+
"gradient_accumulation_steps": 15,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 1000,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": true,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b mixed 8k seed70",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"mixed",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": null,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 35,
|
| 58 |
+
"sparse_mask_pool_size": 10000,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 62 |
+
"sparse_mask_seed": 70,
|
| 63 |
+
"sparse_mixed_nrb_set": [
|
| 64 |
+
63,
|
| 65 |
+
47,
|
| 66 |
+
31,
|
| 67 |
+
21,
|
| 68 |
+
11
|
| 69 |
+
],
|
| 70 |
+
"dense_mode": false,
|
| 71 |
+
"dense_step_intervals": null
|
| 72 |
+
}
|
| 73 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 74 |
+
Initializing model from scratch (random weights)...
|
| 75 |
+
Checkpoint activations: True
|
| 76 |
+
Checkpoint activations: True
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
================================================================================
|
| 80 |
+
MODEL ARCHITECTURE
|
| 81 |
+
================================================================================
|
| 82 |
+
Model config:
|
| 83 |
+
vocab_size: 151936
|
| 84 |
+
hidden_size: 1536
|
| 85 |
+
num_hidden_layers: 28
|
| 86 |
+
num_attention_heads: 12
|
| 87 |
+
num_key_value_heads: 4
|
| 88 |
+
head_dim: 128
|
| 89 |
+
intermediate_size: 6144
|
| 90 |
+
max_position_embeddings: 32768
|
| 91 |
+
rope_theta: 1000000
|
| 92 |
+
rope_train_position_scale_set: None
|
| 93 |
+
attention_type: sparse_stochastic
|
| 94 |
+
sparse_block_size: 128
|
| 95 |
+
sparse_num_random_blocks: 35
|
| 96 |
+
sparse_mask_pool_size: 10000
|
| 97 |
+
sparse_max_seq_length: 8192
|
| 98 |
+
|
| 99 |
+
Parameter count:
|
| 100 |
+
Total parameters: 1,202,352,640
|
| 101 |
+
Trainable parameters: 1,202,352,640
|
| 102 |
+
Non-trainable parameters: 0
|
| 103 |
+
Model size (BF16): 2.40 GB
|
| 104 |
+
|
| 105 |
+
--------------------------------------------------------------------------------
|
| 106 |
+
LAYER-BY-LAYER SUMMARY
|
| 107 |
+
--------------------------------------------------------------------------------
|
| 108 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 109 |
+
|
| 110 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 111 |
+
|
| 112 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 113 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 114 |
+
|
| 115 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 116 |
+
_warn_once(
|
| 117 |
+
================================================================================================================================================================
|
| 118 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 119 |
+
================================================================================================================================================================
|
| 120 |
+
Qwen3ForCausalLM [2, 8192] -- -- True
|
| 121 |
+
├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True
|
| 122 |
+
│ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True
|
| 123 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 124 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 125 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 126 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True
|
| 153 |
+
├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True
|
| 154 |
+
================================================================================================================================================================
|
| 155 |
+
Total params: 1,435,726,336
|
| 156 |
+
Trainable params: 1,435,726,336
|
| 157 |
+
Non-trainable params: 0
|
| 158 |
+
Total mult-adds (Units.GIGABYTES): 2.87
|
| 159 |
+
================================================================================================================================================================
|
| 160 |
+
Input size (MB): 0.13
|
| 161 |
+
Forward/backward pass size (MB): 52437.19
|
| 162 |
+
Params size (MB): 2871.45
|
| 163 |
+
Estimated Total Size (MB): 55308.77
|
| 164 |
+
================================================================================================================================================================
|
| 165 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 166 |
+
================================================================================
|
| 167 |
+
|
| 168 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 169 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 170 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 171 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 172 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 173 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 174 |
+
Dataset split: 3027764 train, 159356 validation
|
| 175 |
+
Training examples (packed, train split): 3027764
|
| 176 |
+
Validation examples (packed, val split): 159356
|
| 177 |
+
Per-device batch size: 2
|
| 178 |
+
World size (GPUs): 2
|
| 179 |
+
Gradient accumulation steps: 15
|
| 180 |
+
Effective batch size (seqs/step): 60
|
| 181 |
+
Sequence length: 8192
|
| 182 |
+
Tokens per optimizer step: 491,520
|
| 183 |
+
Micro-batches per epoch (per rank): 756941
|
| 184 |
+
Optimizer steps per epoch: 50462
|
| 185 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 186 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 187 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 188 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 189 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260822_210348-i8ewyy95
|
| 190 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 191 |
+
wandb: Syncing run 1.2b mixed 8k seed70
|
| 192 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 193 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i8ewyy95
|
| 194 |
+
Starting training...
|
| 195 |
+
Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
|
| 196 |
+
Step 10/50462 | Loss: 12.2285 | LR: 6.67e-06 | 13558.2ms/step | 4.4 samples/s
|
| 197 |
+
Step 20/50462 | Loss: 11.9217 | LR: 1.33e-05 | 7347.1ms/step | 8.2 samples/s
|
| 198 |
+
Step 30/50462 | Loss: 11.0220 | LR: 2.00e-05 | 7347.0ms/step | 8.2 samples/s
|
| 199 |
+
Step 40/50462 | Loss: 10.1514 | LR: 2.67e-05 | 7315.7ms/step | 8.2 samples/s
|
| 200 |
+
Step 50/50462 | Loss: 9.7299 | LR: 3.33e-05 | 7338.4ms/step | 8.2 samples/s
|
| 201 |
+
Step 60/50462 | Loss: 9.2932 | LR: 4.00e-05 | 7295.1ms/step | 8.2 samples/s
|
| 202 |
+
Step 70/50462 | Loss: 8.7279 | LR: 4.67e-05 | 7344.8ms/step | 8.2 samples/s
|
| 203 |
+
Step 80/50462 | Loss: 8.2714 | LR: 5.33e-05 | 7343.8ms/step | 8.2 samples/s
|
| 204 |
+
Step 90/50462 | Loss: 7.8736 | LR: 6.00e-05 | 7312.5ms/step | 8.2 samples/s
|
| 205 |
+
Step 100/50462 | Loss: 7.4990 | LR: 6.67e-05 | 7328.6ms/step | 8.2 samples/s
|
| 206 |
+
Step 110/50462 | Loss: 7.2479 | LR: 7.33e-05 | 7341.2ms/step | 8.2 samples/s
|
| 207 |
+
Step 120/50462 | Loss: 7.0941 | LR: 8.00e-05 | 7335.0ms/step | 8.2 samples/s
|
| 208 |
+
Step 130/50462 | Loss: 6.9836 | LR: 8.67e-05 | 7345.7ms/step | 8.2 samples/s
|
| 209 |
+
Step 140/50462 | Loss: 6.8738 | LR: 9.33e-05 | 7361.3ms/step | 8.2 samples/s
|
| 210 |
+
Step 150/50462 | Loss: 6.7698 | LR: 1.00e-04 | 7359.0ms/step | 8.2 samples/s
|
| 211 |
+
Step 160/50462 | Loss: 6.6554 | LR: 1.07e-04 | 7384.1ms/step | 8.1 samples/s
|
| 212 |
+
Step 170/50462 | Loss: 6.5238 | LR: 1.13e-04 | 7386.8ms/step | 8.1 samples/s
|
| 213 |
+
Step 180/50462 | Loss: 6.4588 | LR: 1.20e-04 | 7359.1ms/step | 8.2 samples/s
|
| 214 |
+
Step 190/50462 | Loss: 6.3743 | LR: 1.27e-04 | 7343.1ms/step | 8.2 samples/s
|
| 215 |
+
Step 200/50462 | Loss: 6.2806 | LR: 1.33e-04 | 7351.1ms/step | 8.2 samples/s
|
| 216 |
+
Step 210/50462 | Loss: 6.2474 | LR: 1.40e-04 | 7363.2ms/step | 8.1 samples/s
|
| 217 |
+
Step 220/50462 | Loss: 6.2108 | LR: 1.47e-04 | 7324.9ms/step | 8.2 samples/s
|
| 218 |
+
Step 230/50462 | Loss: 6.1300 | LR: 1.53e-04 | 7372.7ms/step | 8.1 samples/s
|
| 219 |
+
Step 240/50462 | Loss: 6.0595 | LR: 1.60e-04 | 7399.2ms/step | 8.1 samples/s
|
| 220 |
+
Step 250/50462 | Loss: 6.0046 | LR: 1.67e-04 | 7362.8ms/step | 8.1 samples/s
|
| 221 |
+
Validation at step 250: Loss = 5.9596, Perplexity = 387.45
|
| 222 |
+
Step 260/50462 | Loss: 5.9345 | LR: 1.73e-04 | 15108.8ms/step | 4.0 samples/s
|
| 223 |
+
Step 270/50462 | Loss: 5.8871 | LR: 1.80e-04 | 7355.1ms/step | 8.2 samples/s
|
| 224 |
+
Step 280/50462 | Loss: 5.8307 | LR: 1.87e-04 | 7401.4ms/step | 8.1 samples/s
|
| 225 |
+
Step 290/50462 | Loss: 5.7848 | LR: 1.93e-04 | 7368.6ms/step | 8.1 samples/s
|
| 226 |
+
Step 300/50462 | Loss: 5.7138 | LR: 2.00e-04 | 7369.7ms/step | 8.1 samples/s
|
| 227 |
+
Step 310/50462 | Loss: 5.6581 | LR: 2.07e-04 | 7373.0ms/step | 8.1 samples/s
|
| 228 |
+
Step 320/50462 | Loss: 5.6094 | LR: 2.13e-04 | 7385.3ms/step | 8.1 samples/s
|
| 229 |
+
Step 330/50462 | Loss: 5.5933 | LR: 2.20e-04 | 7341.1ms/step | 8.2 samples/s
|
| 230 |
+
Step 340/50462 | Loss: 5.5433 | LR: 2.27e-04 | 7334.3ms/step | 8.2 samples/s
|
| 231 |
+
Step 350/50462 | Loss: 5.4962 | LR: 2.33e-04 | 7377.7ms/step | 8.1 samples/s
|
| 232 |
+
Step 360/50462 | Loss: 5.4527 | LR: 2.40e-04 | 7381.3ms/step | 8.1 samples/s
|
| 233 |
+
Step 370/50462 | Loss: 5.3898 | LR: 2.47e-04 | 7373.7ms/step | 8.1 samples/s
|
| 234 |
+
Step 380/50462 | Loss: 5.3137 | LR: 2.53e-04 | 7353.7ms/step | 8.2 samples/s
|
| 235 |
+
Step 390/50462 | Loss: 5.2732 | LR: 2.60e-04 | 7360.9ms/step | 8.2 samples/s
|
| 236 |
+
Step 400/50462 | Loss: 5.2140 | LR: 2.67e-04 | 7363.3ms/step | 8.1 samples/s
|
| 237 |
+
Step 410/50462 | Loss: 5.2023 | LR: 2.73e-04 | 7377.4ms/step | 8.1 samples/s
|
| 238 |
+
Step 420/50462 | Loss: 5.1795 | LR: 2.80e-04 | 7380.2ms/step | 8.1 samples/s
|
| 239 |
+
Step 430/50462 | Loss: 5.1245 | LR: 2.87e-04 | 7372.2ms/step | 8.1 samples/s
|
| 240 |
+
Step 440/50462 | Loss: 5.0656 | LR: 2.93e-04 | 7356.0ms/step | 8.2 samples/s
|
| 241 |
+
Step 450/50462 | Loss: 5.0408 | LR: 3.00e-04 | 7371.9ms/step | 8.1 samples/s
|
| 242 |
+
Step 460/50462 | Loss: 5.0109 | LR: 3.07e-04 | 7383.0ms/step | 8.1 samples/s
|
| 243 |
+
Step 470/50462 | Loss: 5.0111 | LR: 3.13e-04 | 7364.5ms/step | 8.1 samples/s
|
| 244 |
+
Step 480/50462 | Loss: 4.9881 | LR: 3.20e-04 | 7383.8ms/step | 8.1 samples/s
|
| 245 |
+
Step 490/50462 | Loss: 4.8550 | LR: 3.27e-04 | 7361.0ms/step | 8.2 samples/s
|
| 246 |
+
Step 500/50462 | Loss: 4.8979 | LR: 3.33e-04 | 7376.8ms/step | 8.1 samples/s
|
| 247 |
+
Validation at step 500: Loss = 4.9163, Perplexity = 136.50
|
| 248 |
+
Step 510/50462 | Loss: 4.9215 | LR: 3.40e-04 | 11221.6ms/step | 5.3 samples/s
|
| 249 |
+
Step 520/50462 | Loss: 4.8483 | LR: 3.47e-04 | 7382.2ms/step | 8.1 samples/s
|
| 250 |
+
Step 530/50462 | Loss: 4.7789 | LR: 3.53e-04 | 7335.3ms/step | 8.2 samples/s
|
| 251 |
+
Step 540/50462 | Loss: 4.7686 | LR: 3.60e-04 | 7365.3ms/step | 8.1 samples/s
|
| 252 |
+
Step 550/50462 | Loss: 4.8117 | LR: 3.67e-04 | 7364.1ms/step | 8.1 samples/s
|
| 253 |
+
Step 560/50462 | Loss: 4.7064 | LR: 3.73e-04 | 7379.0ms/step | 8.1 samples/s
|
| 254 |
+
Step 570/50462 | Loss: 4.6854 | LR: 3.80e-04 | 7355.6ms/step | 8.2 samples/s
|
| 255 |
+
Step 580/50462 | Loss: 4.6726 | LR: 3.87e-04 | 7331.4ms/step | 8.2 samples/s
|
| 256 |
+
Step 590/50462 | Loss: 4.5973 | LR: 3.93e-04 | 7381.6ms/step | 8.1 samples/s
|
| 257 |
+
Step 600/50462 | Loss: 4.5529 | LR: 4.00e-04 | 7349.9ms/step | 8.2 samples/s
|
| 258 |
+
Step 610/50462 | Loss: 4.6087 | LR: 4.07e-04 | 7334.5ms/step | 8.2 samples/s
|
| 259 |
+
Step 620/50462 | Loss: 4.6407 | LR: 4.13e-04 | 7371.9ms/step | 8.1 samples/s
|
| 260 |
+
Step 630/50462 | Loss: 4.5022 | LR: 4.20e-04 | 7390.2ms/step | 8.1 samples/s
|
| 261 |
+
Step 640/50462 | Loss: 4.4227 | LR: 4.27e-04 | 7391.0ms/step | 8.1 samples/s
|
| 262 |
+
Step 650/50462 | Loss: 4.4289 | LR: 4.33e-04 | 7363.4ms/step | 8.1 samples/s
|
| 263 |
+
Step 660/50462 | Loss: 4.4917 | LR: 4.40e-04 | 7351.9ms/step | 8.2 samples/s
|
| 264 |
+
Step 670/50462 | Loss: 4.4139 | LR: 4.47e-04 | 7390.2ms/step | 8.1 samples/s
|
| 265 |
+
Step 680/50462 | Loss: 4.3439 | LR: 4.53e-04 | 7400.2ms/step | 8.1 samples/s
|
| 266 |
+
Step 690/50462 | Loss: 4.2979 | LR: 4.60e-04 | 7343.6ms/step | 8.2 samples/s
|
| 267 |
+
Step 700/50462 | Loss: 4.3337 | LR: 4.67e-04 | 7367.1ms/step | 8.1 samples/s
|
| 268 |
+
Step 710/50462 | Loss: 4.3510 | LR: 4.73e-04 | 7343.7ms/step | 8.2 samples/s
|
| 269 |
+
Step 720/50462 | Loss: 4.3167 | LR: 4.80e-04 | 7353.9ms/step | 8.2 samples/s
|
| 270 |
+
Step 730/50462 | Loss: 4.2202 | LR: 4.87e-04 | 7370.9ms/step | 8.1 samples/s
|
| 271 |
+
Step 740/50462 | Loss: 4.1858 | LR: 4.93e-04 | 7374.4ms/step | 8.1 samples/s
|
| 272 |
+
Step 750/50462 | Loss: 4.2042 | LR: 5.00e-04 | 7381.0ms/step | 8.1 samples/s
|
| 273 |
+
Validation at step 750: Loss = 4.1897, Perplexity = 66.01
|
| 274 |
+
Step 760/50462 | Loss: 4.1547 | LR: 5.00e-04 | 11212.7ms/step | 5.4 samples/s
|
| 275 |
+
Step 770/50462 | Loss: 4.1554 | LR: 5.00e-04 | 7374.2ms/step | 8.1 samples/s
|
| 276 |
+
Step 780/50462 | Loss: 4.1212 | LR: 5.00e-04 | 7378.7ms/step | 8.1 samples/s
|
| 277 |
+
Step 790/50462 | Loss: 4.0891 | LR: 5.00e-04 | 7378.7ms/step | 8.1 samples/s
|
| 278 |
+
Step 800/50462 | Loss: 4.0808 | LR: 5.00e-04 | 7383.1ms/step | 8.1 samples/s
|
| 279 |
+
Step 810/50462 | Loss: 4.0226 | LR: 5.00e-04 | 7397.9ms/step | 8.1 samples/s
|
| 280 |
+
Step 820/50462 | Loss: 4.0400 | LR: 5.00e-04 | 7372.5ms/step | 8.1 samples/s
|
| 281 |
+
Step 830/50462 | Loss: 4.0047 | LR: 5.00e-04 | 7357.1ms/step | 8.2 samples/s
|
| 282 |
+
Step 840/50462 | Loss: 4.0164 | LR: 5.00e-04 | 7380.1ms/step | 8.1 samples/s
|
| 283 |
+
Step 850/50462 | Loss: 3.9565 | LR: 5.00e-04 | 7381.0ms/step | 8.1 samples/s
|
| 284 |
+
Step 860/50462 | Loss: 3.9267 | LR: 5.00e-04 | 7365.3ms/step | 8.1 samples/s
|
| 285 |
+
Step 870/50462 | Loss: 3.9078 | LR: 5.00e-04 | 7375.7ms/step | 8.1 samples/s
|
| 286 |
+
Step 880/50462 | Loss: 3.9065 | LR: 5.00e-04 | 7360.9ms/step | 8.2 samples/s
|
| 287 |
+
Step 890/50462 | Loss: 3.9025 | LR: 5.00e-04 | 7364.2ms/step | 8.1 samples/s
|
| 288 |
+
Step 900/50462 | Loss: 3.8937 | LR: 5.00e-04 | 7341.4ms/step | 8.2 samples/s
|
| 289 |
+
Step 910/50462 | Loss: 3.8811 | LR: 5.00e-04 | 7388.4ms/step | 8.1 samples/s
|
| 290 |
+
Step 920/50462 | Loss: 3.8701 | LR: 5.00e-04 | 7363.6ms/step | 8.1 samples/s
|
| 291 |
+
Step 930/50462 | Loss: 3.8368 | LR: 5.00e-04 | 7377.5ms/step | 8.1 samples/s
|
| 292 |
+
Step 940/50462 | Loss: 3.8551 | LR: 5.00e-04 | 7390.4ms/step | 8.1 samples/s
|
| 293 |
+
Step 950/50462 | Loss: 3.8354 | LR: 5.00e-04 | 7345.6ms/step | 8.2 samples/s
|
| 294 |
+
Step 960/50462 | Loss: 3.7922 | LR: 5.00e-04 | 7367.2ms/step | 8.1 samples/s
|
| 295 |
+
Step 970/50462 | Loss: 3.8038 | LR: 5.00e-04 | 7372.8ms/step | 8.1 samples/s
|
| 296 |
+
Step 980/50462 | Loss: 3.7930 | LR: 5.00e-04 | 7374.7ms/step | 8.1 samples/s
|
| 297 |
+
Step 990/50462 | Loss: 3.7858 | LR: 5.00e-04 | 7361.0ms/step | 8.2 samples/s
|
| 298 |
+
Step 1000/50462 | Loss: 3.7586 | LR: 5.00e-04 | 7394.0ms/step | 8.1 samples/s
|
| 299 |
+
Validation at step 1000: Loss = 3.7727, Perplexity = 43.50
|
| 300 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-1000
|
| 301 |
+
Step 1010/50462 | Loss: 3.7469 | LR: 5.00e-04 | 12122.1ms/step | 4.9 samples/s
|
| 302 |
+
Step 1020/50462 | Loss: 3.7524 | LR: 5.00e-04 | 7364.7ms/step | 8.1 samples/s
|
| 303 |
+
Step 1030/50462 | Loss: 3.7150 | LR: 5.00e-04 | 7356.9ms/step | 8.2 samples/s
|
| 304 |
+
Step 1040/50462 | Loss: 3.7204 | LR: 5.00e-04 | 7359.6ms/step | 8.2 samples/s
|
| 305 |
+
Step 1050/50462 | Loss: 3.7048 | LR: 5.00e-04 | 7369.4ms/step | 8.1 samples/s
|
| 306 |
+
Step 1060/50462 | Loss: 3.7026 | LR: 5.00e-04 | 7397.3ms/step | 8.1 samples/s
|
| 307 |
+
Step 1070/50462 | Loss: 3.6855 | LR: 5.00e-04 | 7386.1ms/step | 8.1 samples/s
|
| 308 |
+
Step 1080/50462 | Loss: 3.6904 | LR: 5.00e-04 | 7365.9ms/step | 8.1 samples/s
|
| 309 |
+
Step 1090/50462 | Loss: 3.6655 | LR: 5.00e-04 | 7377.9ms/step | 8.1 samples/s
|
| 310 |
+
Step 1100/50462 | Loss: 3.6564 | LR: 5.00e-04 | 7375.4ms/step | 8.1 samples/s
|
| 311 |
+
Step 1110/50462 | Loss: 3.6768 | LR: 5.00e-04 | 7365.5ms/step | 8.1 samples/s
|
| 312 |
+
Step 1120/50462 | Loss: 3.6380 | LR: 5.00e-04 | 7389.6ms/step | 8.1 samples/s
|
| 313 |
+
Step 1130/50462 | Loss: 3.6196 | LR: 5.00e-04 | 7389.6ms/step | 8.1 samples/s
|
| 314 |
+
Step 1140/50462 | Loss: 3.6203 | LR: 5.00e-04 | 7380.5ms/step | 8.1 samples/s
|
| 315 |
+
Step 1150/50462 | Loss: 3.6380 | LR: 5.00e-04 | 7374.1ms/step | 8.1 samples/s
|
| 316 |
+
Step 1160/50462 | Loss: 3.6174 | LR: 5.00e-04 | 7363.0ms/step | 8.1 samples/s
|
| 317 |
+
Step 1170/50462 | Loss: 3.6118 | LR: 5.00e-04 | 7369.1ms/step | 8.1 samples/s
|
| 318 |
+
Step 1180/50462 | Loss: 3.5977 | LR: 5.00e-04 | 7338.4ms/step | 8.2 samples/s
|
| 319 |
+
Step 1190/50462 | Loss: 3.6211 | LR: 5.00e-04 | 7363.7ms/step | 8.1 samples/s
|
| 320 |
+
Step 1200/50462 | Loss: 3.5694 | LR: 5.00e-04 | 7369.7ms/step | 8.1 samples/s
|
| 321 |
+
Step 1210/50462 | Loss: 3.5754 | LR: 5.00e-04 | 7364.3ms/step | 8.1 samples/s
|
| 322 |
+
Step 1220/50462 | Loss: 3.6185 | LR: 5.00e-04 | 7361.4ms/step | 8.2 samples/s
|
| 323 |
+
Step 1230/50462 | Loss: 3.5867 | LR: 5.00e-04 | 7368.8ms/step | 8.1 samples/s
|
| 324 |
+
Step 1240/50462 | Loss: 3.5624 | LR: 5.00e-04 | 7343.2ms/step | 8.2 samples/s
|
| 325 |
+
Step 1250/50462 | Loss: 3.5676 | LR: 5.00e-04 | 7344.9ms/step | 8.2 samples/s
|
| 326 |
+
Validation at step 1250: Loss = 3.5575, Perplexity = 35.08
|
| 327 |
+
Step 1260/50462 | Loss: 3.5516 | LR: 5.00e-04 | 11209.6ms/step | 5.4 samples/s
|
| 328 |
+
Step 1270/50462 | Loss: 3.5104 | LR: 5.00e-04 | 7370.2ms/step | 8.1 samples/s
|
| 329 |
+
Step 1280/50462 | Loss: 3.5620 | LR: 5.00e-04 | 7376.4ms/step | 8.1 samples/s
|
| 330 |
+
Step 1290/50462 | Loss: 3.5158 | LR: 5.00e-04 | 7349.9ms/step | 8.2 samples/s
|
| 331 |
+
Step 1300/50462 | Loss: 3.5342 | LR: 5.00e-04 | 7373.2ms/step | 8.1 samples/s
|
| 332 |
+
Step 1310/50462 | Loss: 3.5230 | LR: 5.00e-04 | 7391.4ms/step | 8.1 samples/s
|
| 333 |
+
Step 1320/50462 | Loss: 3.5181 | LR: 5.00e-04 | 7381.5ms/step | 8.1 samples/s
|
| 334 |
+
Step 1330/50462 | Loss: 3.4818 | LR: 5.00e-04 | 7386.5ms/step | 8.1 samples/s
|
| 335 |
+
Step 1340/50462 | Loss: 3.4871 | LR: 5.00e-04 | 7382.4ms/step | 8.1 samples/s
|
| 336 |
+
Step 1350/50462 | Loss: 3.4923 | LR: 5.00e-04 | 7370.9ms/step | 8.1 samples/s
|
| 337 |
+
Step 1360/50462 | Loss: 3.4964 | LR: 5.00e-04 | 7355.6ms/step | 8.2 samples/s
|
| 338 |
+
Step 1370/50462 | Loss: 3.5066 | LR: 5.00e-04 | 7372.6ms/step | 8.1 samples/s
|
| 339 |
+
Step 1380/50462 | Loss: 3.4892 | LR: 5.00e-04 | 7384.9ms/step | 8.1 samples/s
|
| 340 |
+
Step 1390/50462 | Loss: 3.4640 | LR: 5.00e-04 | 7387.6ms/step | 8.1 samples/s
|
| 341 |
+
Step 1400/50462 | Loss: 3.4667 | LR: 5.00e-04 | 7355.3ms/step | 8.2 samples/s
|
| 342 |
+
Step 1410/50462 | Loss: 3.4724 | LR: 5.00e-04 | 7371.0ms/step | 8.1 samples/s
|
| 343 |
+
Step 1420/50462 | Loss: 3.4577 | LR: 5.00e-04 | 7366.0ms/step | 8.1 samples/s
|
| 344 |
+
Step 1430/50462 | Loss: 3.4689 | LR: 5.00e-04 | 7380.9ms/step | 8.1 samples/s
|
| 345 |
+
Step 1440/50462 | Loss: 3.4861 | LR: 5.00e-04 | 7367.2ms/step | 8.1 samples/s
|
| 346 |
+
Step 1450/50462 | Loss: 3.4154 | LR: 5.00e-04 | 7378.6ms/step | 8.1 samples/s
|
| 347 |
+
Step 1460/50462 | Loss: 3.4327 | LR: 5.00e-04 | 7386.3ms/step | 8.1 samples/s
|
| 348 |
+
Step 1470/50462 | Loss: 3.4261 | LR: 5.00e-04 | 7363.5ms/step | 8.1 samples/s
|
| 349 |
+
Step 1480/50462 | Loss: 3.4263 | LR: 5.00e-04 | 7359.7ms/step | 8.2 samples/s
|
| 350 |
+
Step 1490/50462 | Loss: 3.4337 | LR: 5.00e-04 | 7356.8ms/step | 8.2 samples/s
|
| 351 |
+
Step 1500/50462 | Loss: 3.4186 | LR: 5.00e-04 | 7358.8ms/step | 8.2 samples/s
|
| 352 |
+
Validation at step 1500: Loss = 3.4334, Perplexity = 30.98
|
| 353 |
+
Step 1510/50462 | Loss: 3.3993 | LR: 5.00e-04 | 11241.4ms/step | 5.3 samples/s
|
| 354 |
+
Step 1520/50462 | Loss: 3.3938 | LR: 5.00e-04 | 7390.4ms/step | 8.1 samples/s
|
| 355 |
+
Step 1530/50462 | Loss: 3.4041 | LR: 5.00e-04 | 7363.7ms/step | 8.1 samples/s
|
| 356 |
+
Step 1540/50462 | Loss: 3.4045 | LR: 5.00e-04 | 7373.7ms/step | 8.1 samples/s
|
| 357 |
+
Step 1550/50462 | Loss: 3.4000 | LR: 5.00e-04 | 7353.9ms/step | 8.2 samples/s
|
| 358 |
+
Step 1560/50462 | Loss: 3.4077 | LR: 5.00e-04 | 7359.4ms/step | 8.2 samples/s
|
| 359 |
+
Step 1570/50462 | Loss: 3.4002 | LR: 5.00e-04 | 7372.3ms/step | 8.1 samples/s
|
| 360 |
+
Step 1580/50462 | Loss: 3.3963 | LR: 5.00e-04 | 7343.2ms/step | 8.2 samples/s
|
| 361 |
+
Step 1590/50462 | Loss: 3.3845 | LR: 5.00e-04 | 7337.8ms/step | 8.2 samples/s
|
| 362 |
+
Step 1600/50462 | Loss: 3.4028 | LR: 5.00e-04 | 7377.2ms/step | 8.1 samples/s
|
| 363 |
+
Step 1610/50462 | Loss: 3.3927 | LR: 5.00e-04 | 7372.9ms/step | 8.1 samples/s
|
| 364 |
+
Step 1620/50462 | Loss: 3.3785 | LR: 5.00e-04 | 7391.5ms/step | 8.1 samples/s
|
| 365 |
+
Step 1630/50462 | Loss: 3.3889 | LR: 5.00e-04 | 7370.4ms/step | 8.1 samples/s
|
| 366 |
+
Step 1640/50462 | Loss: 3.3532 | LR: 5.00e-04 | 7387.5ms/step | 8.1 samples/s
|
| 367 |
+
Step 1650/50462 | Loss: 3.3588 | LR: 5.00e-04 | 7345.7ms/step | 8.2 samples/s
|
| 368 |
+
Step 1660/50462 | Loss: 3.3696 | LR: 5.00e-04 | 7386.8ms/step | 8.1 samples/s
|
| 369 |
+
Step 1670/50462 | Loss: 3.3443 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
|
| 370 |
+
Step 1680/50462 | Loss: 3.3622 | LR: 5.00e-04 | 7370.5ms/step | 8.1 samples/s
|
| 371 |
+
Step 1690/50462 | Loss: 3.3558 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
|
| 372 |
+
Step 1700/50462 | Loss: 3.3337 | LR: 5.00e-04 | 7360.3ms/step | 8.2 samples/s
|
| 373 |
+
Step 1710/50462 | Loss: 3.3476 | LR: 5.00e-04 | 7362.4ms/step | 8.1 samples/s
|
| 374 |
+
Step 1720/50462 | Loss: 3.3274 | LR: 5.00e-04 | 7381.6ms/step | 8.1 samples/s
|
| 375 |
+
Step 1730/50462 | Loss: 3.3361 | LR: 5.00e-04 | 7371.6ms/step | 8.1 samples/s
|
| 376 |
+
Step 1740/50462 | Loss: 3.3073 | LR: 5.00e-04 | 7363.3ms/step | 8.1 samples/s
|
| 377 |
+
Step 1750/50462 | Loss: 3.3367 | LR: 5.00e-04 | 7366.8ms/step | 8.1 samples/s
|
| 378 |
+
Validation at step 1750: Loss = 3.3428, Perplexity = 28.30
|
| 379 |
+
Step 1760/50462 | Loss: 3.3488 | LR: 4.99e-04 | 11240.3ms/step | 5.3 samples/s
|
| 380 |
+
Step 1770/50462 | Loss: 3.3295 | LR: 4.99e-04 | 7368.6ms/step | 8.1 samples/s
|
| 381 |
+
Step 1780/50462 | Loss: 3.3044 | LR: 4.99e-04 | 7381.3ms/step | 8.1 samples/s
|
| 382 |
+
Step 1790/50462 | Loss: 3.3265 | LR: 4.99e-04 | 7351.5ms/step | 8.2 samples/s
|
| 383 |
+
Step 1800/50462 | Loss: 3.3426 | LR: 4.99e-04 | 7354.6ms/step | 8.2 samples/s
|
| 384 |
+
Step 1810/50462 | Loss: 3.3453 | LR: 4.99e-04 | 7364.2ms/step | 8.1 samples/s
|
| 385 |
+
Step 1820/50462 | Loss: 3.3235 | LR: 4.99e-04 | 7377.3ms/step | 8.1 samples/s
|
| 386 |
+
Step 1830/50462 | Loss: 3.3117 | LR: 4.99e-04 | 7371.0ms/step | 8.1 samples/s
|
| 387 |
+
Step 1840/50462 | Loss: 3.2834 | LR: 4.99e-04 | 7374.8ms/step | 8.1 samples/s
|
| 388 |
+
Step 1850/50462 | Loss: 3.3039 | LR: 4.99e-04 | 7383.4ms/step | 8.1 samples/s
|
| 389 |
+
Step 1860/50462 | Loss: 3.2982 | LR: 4.99e-04 | 7386.3ms/step | 8.1 samples/s
|
| 390 |
+
Step 1870/50462 | Loss: 3.3197 | LR: 4.99e-04 | 7362.8ms/step | 8.1 samples/s
|
| 391 |
+
Step 1880/50462 | Loss: 3.3018 | LR: 4.99e-04 | 7345.8ms/step | 8.2 samples/s
|
| 392 |
+
Step 1890/50462 | Loss: 3.2601 | LR: 4.99e-04 | 7371.6ms/step | 8.1 samples/s
|
| 393 |
+
Step 1900/50462 | Loss: 3.2972 | LR: 4.99e-04 | 7377.8ms/step | 8.1 samples/s
|
| 394 |
+
Step 1910/50462 | Loss: 3.2853 | LR: 4.99e-04 | 7369.0ms/step | 8.1 samples/s
|
| 395 |
+
Step 1920/50462 | Loss: 3.2891 | LR: 4.99e-04 | 7334.5ms/step | 8.2 samples/s
|
| 396 |
+
Step 1930/50462 | Loss: 3.2646 | LR: 4.99e-04 | 7325.2ms/step | 8.2 samples/s
|
| 397 |
+
Step 1940/50462 | Loss: 3.2892 | LR: 4.99e-04 | 7362.3ms/step | 8.1 samples/s
|
| 398 |
+
Step 1950/50462 | Loss: 3.2870 | LR: 4.99e-04 | 7350.6ms/step | 8.2 samples/s
|
| 399 |
+
Step 1960/50462 | Loss: 3.2640 | LR: 4.99e-04 | 7379.5ms/step | 8.1 samples/s
|
| 400 |
+
Step 1970/50462 | Loss: 3.2668 | LR: 4.99e-04 | 7381.9ms/step | 8.1 samples/s
|
| 401 |
+
Step 1980/50462 | Loss: 3.2723 | LR: 4.99e-04 | 7371.1ms/step | 8.1 samples/s
|
| 402 |
+
Step 1990/50462 | Loss: 3.2744 | LR: 4.99e-04 | 7357.0ms/step | 8.2 samples/s
|
| 403 |
+
Step 2000/50462 | Loss: 3.2649 | LR: 4.99e-04 | 7394.0ms/step | 8.1 samples/s
|
| 404 |
+
Validation at step 2000: Loss = 3.2750, Perplexity = 26.44
|
| 405 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-2000
|
| 406 |
+
Step 2010/50462 | Loss: 3.2365 | LR: 4.99e-04 | 12182.3ms/step | 4.9 samples/s
|
| 407 |
+
Step 2020/50462 | Loss: 3.2684 | LR: 4.99e-04 | 7350.3ms/step | 8.2 samples/s
|
| 408 |
+
Step 2030/50462 | Loss: 3.2474 | LR: 4.99e-04 | 7358.6ms/step | 8.2 samples/s
|
| 409 |
+
Step 2040/50462 | Loss: 3.2492 | LR: 4.99e-04 | 7387.8ms/step | 8.1 samples/s
|
| 410 |
+
Step 2050/50462 | Loss: 3.2074 | LR: 4.99e-04 | 7398.8ms/step | 8.1 samples/s
|
| 411 |
+
Step 2060/50462 | Loss: 3.2730 | LR: 4.99e-04 | 7362.4ms/step | 8.1 samples/s
|
| 412 |
+
Step 2070/50462 | Loss: 3.2711 | LR: 4.99e-04 | 7352.6ms/step | 8.2 samples/s
|
| 413 |
+
Step 2080/50462 | Loss: 3.2437 | LR: 4.99e-04 | 7358.6ms/step | 8.2 samples/s
|
| 414 |
+
Step 2090/50462 | Loss: 3.2618 | LR: 4.99e-04 | 7356.2ms/step | 8.2 samples/s
|
| 415 |
+
Step 2100/50462 | Loss: 3.2509 | LR: 4.99e-04 | 7385.5ms/step | 8.1 samples/s
|
| 416 |
+
Step 2110/50462 | Loss: 3.2308 | LR: 4.99e-04 | 7381.3ms/step | 8.1 samples/s
|
| 417 |
+
Step 2120/50462 | Loss: 3.2274 | LR: 4.99e-04 | 7387.9ms/step | 8.1 samples/s
|
| 418 |
+
Step 2130/50462 | Loss: 3.2482 | LR: 4.99e-04 | 7376.4ms/step | 8.1 samples/s
|
| 419 |
+
Step 2140/50462 | Loss: 3.2243 | LR: 4.99e-04 | 7374.2ms/step | 8.1 samples/s
|
| 420 |
+
Step 2150/50462 | Loss: 3.2249 | LR: 4.99e-04 | 7367.1ms/step | 8.1 samples/s
|
| 421 |
+
Step 2160/50462 | Loss: 3.2319 | LR: 4.99e-04 | 7336.0ms/step | 8.2 samples/s
|
| 422 |
+
Step 2170/50462 | Loss: 3.2290 | LR: 4.99e-04 | 7345.3ms/step | 8.2 samples/s
|
| 423 |
+
Step 2180/50462 | Loss: 3.2386 | LR: 4.99e-04 | 7376.9ms/step | 8.1 samples/s
|
| 424 |
+
Step 2190/50462 | Loss: 3.2052 | LR: 4.99e-04 | 7368.0ms/step | 8.1 samples/s
|
| 425 |
+
Step 2200/50462 | Loss: 3.2152 | LR: 4.99e-04 | 7360.8ms/step | 8.2 samples/s
|
| 426 |
+
Step 2210/50462 | Loss: 3.2154 | LR: 4.99e-04 | 7399.2ms/step | 8.1 samples/s
|
| 427 |
+
Step 2220/50462 | Loss: 3.2344 | LR: 4.99e-04 | 7379.5ms/step | 8.1 samples/s
|
| 428 |
+
Step 2230/50462 | Loss: 3.2164 | LR: 4.99e-04 | 7384.7ms/step | 8.1 samples/s
|
| 429 |
+
Step 2240/50462 | Loss: 3.2290 | LR: 4.99e-04 | 7360.1ms/step | 8.2 samples/s
|
| 430 |
+
Step 2250/50462 | Loss: 3.2164 | LR: 4.99e-04 | 7361.5ms/step | 8.2 samples/s
|
| 431 |
+
Validation at step 2250: Loss = 3.2215, Perplexity = 25.07
|
| 432 |
+
Step 2260/50462 | Loss: 3.2159 | LR: 4.99e-04 | 11245.4ms/step | 5.3 samples/s
|
| 433 |
+
Step 2270/50462 | Loss: 3.2168 | LR: 4.99e-04 | 7374.4ms/step | 8.1 samples/s
|
| 434 |
+
Step 2280/50462 | Loss: 3.2215 | LR: 4.99e-04 | 7384.3ms/step | 8.1 samples/s
|
| 435 |
+
Step 2290/50462 | Loss: 3.2073 | LR: 4.99e-04 | 7379.9ms/step | 8.1 samples/s
|
| 436 |
+
Step 2300/50462 | Loss: 3.2177 | LR: 4.99e-04 | 7398.5ms/step | 8.1 samples/s
|
| 437 |
+
Step 2310/50462 | Loss: 3.2115 | LR: 4.99e-04 | 7360.8ms/step | 8.2 samples/s
|
| 438 |
+
Step 2320/50462 | Loss: 3.2178 | LR: 4.99e-04 | 7378.7ms/step | 8.1 samples/s
|
| 439 |
+
Step 2330/50462 | Loss: 3.1953 | LR: 4.99e-04 | 7378.5ms/step | 8.1 samples/s
|
| 440 |
+
Step 2340/50462 | Loss: 3.2209 | LR: 4.99e-04 | 7343.1ms/step | 8.2 samples/s
|
| 441 |
+
Step 2350/50462 | Loss: 3.1798 | LR: 4.99e-04 | 7357.7ms/step | 8.2 samples/s
|
| 442 |
+
Step 2360/50462 | Loss: 3.1912 | LR: 4.99e-04 | 7357.5ms/step | 8.2 samples/s
|
| 443 |
+
Step 2370/50462 | Loss: 3.2037 | LR: 4.99e-04 | 7364.4ms/step | 8.1 samples/s
|
| 444 |
+
Step 2380/50462 | Loss: 3.1862 | LR: 4.99e-04 | 7359.4ms/step | 8.2 samples/s
|
| 445 |
+
Step 2390/50462 | Loss: 3.2056 | LR: 4.99e-04 | 7380.6ms/step | 8.1 samples/s
|
| 446 |
+
Step 2400/50462 | Loss: 3.2027 | LR: 4.99e-04 | 7377.0ms/step | 8.1 samples/s
|
| 447 |
+
Step 2410/50462 | Loss: 3.1688 | LR: 4.99e-04 | 7356.0ms/step | 8.2 samples/s
|
| 448 |
+
Step 2420/50462 | Loss: 3.1642 | LR: 4.99e-04 | 7364.2ms/step | 8.1 samples/s
|
| 449 |
+
Step 2430/50462 | Loss: 3.1793 | LR: 4.99e-04 | 7400.6ms/step | 8.1 samples/s
|
| 450 |
+
Step 2440/50462 | Loss: 3.1719 | LR: 4.99e-04 | 7396.7ms/step | 8.1 samples/s
|
| 451 |
+
Step 2450/50462 | Loss: 3.1869 | LR: 4.99e-04 | 7373.8ms/step | 8.1 samples/s
|
| 452 |
+
Step 2460/50462 | Loss: 3.1767 | LR: 4.99e-04 | 7387.4ms/step | 8.1 samples/s
|
| 453 |
+
Step 2470/50462 | Loss: 3.1712 | LR: 4.99e-04 | 7355.1ms/step | 8.2 samples/s
|
| 454 |
+
Step 2480/50462 | Loss: 3.1618 | LR: 4.99e-04 | 7382.6ms/step | 8.1 samples/s
|
| 455 |
+
Step 2490/50462 | Loss: 3.1676 | LR: 4.98e-04 | 7389.2ms/step | 8.1 samples/s
|
| 456 |
+
Step 2500/50462 | Loss: 3.1875 | LR: 4.98e-04 | 7374.6ms/step | 8.1 samples/s
|
| 457 |
+
Validation at step 2500: Loss = 3.1794, Perplexity = 24.03
|
| 458 |
+
Step 2510/50462 | Loss: 3.1607 | LR: 4.98e-04 | 11242.0ms/step | 5.3 samples/s
|
| 459 |
+
Step 2520/50462 | Loss: 3.1564 | LR: 4.98e-04 | 7383.3ms/step | 8.1 samples/s
|
| 460 |
+
Step 2530/50462 | Loss: 3.1756 | LR: 4.98e-04 | 7379.6ms/step | 8.1 samples/s
|
| 461 |
+
Step 2540/50462 | Loss: 3.1665 | LR: 4.98e-04 | 7360.2ms/step | 8.2 samples/s
|
| 462 |
+
Step 2550/50462 | Loss: 3.1647 | LR: 4.98e-04 | 7362.0ms/step | 8.2 samples/s
|
| 463 |
+
Step 2560/50462 | Loss: 3.1507 | LR: 4.98e-04 | 7372.0ms/step | 8.1 samples/s
|
| 464 |
+
Step 2570/50462 | Loss: 3.1585 | LR: 4.98e-04 | 7358.7ms/step | 8.2 samples/s
|
| 465 |
+
Step 2580/50462 | Loss: 3.1548 | LR: 4.98e-04 | 7358.4ms/step | 8.2 samples/s
|
| 466 |
+
Step 2590/50462 | Loss: 3.1384 | LR: 4.98e-04 | 7380.4ms/step | 8.1 samples/s
|
| 467 |
+
Step 2600/50462 | Loss: 3.1661 | LR: 4.98e-04 | 7371.2ms/step | 8.1 samples/s
|
| 468 |
+
Step 2610/50462 | Loss: 3.1530 | LR: 4.98e-04 | 7399.3ms/step | 8.1 samples/s
|
| 469 |
+
Step 2620/50462 | Loss: 3.1696 | LR: 4.98e-04 | 7373.0ms/step | 8.1 samples/s
|
| 470 |
+
Step 2630/50462 | Loss: 3.1277 | LR: 4.98e-04 | 7367.1ms/step | 8.1 samples/s
|
| 471 |
+
Step 2640/50462 | Loss: 3.1604 | LR: 4.98e-04 | 7375.0ms/step | 8.1 samples/s
|
| 472 |
+
Step 2650/50462 | Loss: 3.1427 | LR: 4.98e-04 | 7388.8ms/step | 8.1 samples/s
|
| 473 |
+
Step 2660/50462 | Loss: 3.1738 | LR: 4.98e-04 | 7347.5ms/step | 8.2 samples/s
|
| 474 |
+
Step 2670/50462 | Loss: 3.1324 | LR: 4.98e-04 | 7392.1ms/step | 8.1 samples/s
|
| 475 |
+
Step 2680/50462 | Loss: 3.1455 | LR: 4.98e-04 | 7374.6ms/step | 8.1 samples/s
|
| 476 |
+
Step 2690/50462 | Loss: 3.1413 | LR: 4.98e-04 | 7373.6ms/step | 8.1 samples/s
|
| 477 |
+
Step 2700/50462 | Loss: 3.1465 | LR: 4.98e-04 | 7378.4ms/step | 8.1 samples/s
|
| 478 |
+
Step 2710/50462 | Loss: 3.1472 | LR: 4.98e-04 | 7352.7ms/step | 8.2 samples/s
|
| 479 |
+
Step 2720/50462 | Loss: 3.1333 | LR: 4.98e-04 | 7373.7ms/step | 8.1 samples/s
|
| 480 |
+
Step 2730/50462 | Loss: 3.1362 | LR: 4.98e-04 | 7356.3ms/step | 8.2 samples/s
|
| 481 |
+
Step 2740/50462 | Loss: 3.1527 | LR: 4.98e-04 | 7366.7ms/step | 8.1 samples/s
|
| 482 |
+
Step 2750/50462 | Loss: 3.1580 | LR: 4.98e-04 | 7362.4ms/step | 8.1 samples/s
|
| 483 |
+
Validation at step 2750: Loss = 3.1435, Perplexity = 23.18
|
| 484 |
+
Step 2760/50462 | Loss: 3.1579 | LR: 4.98e-04 | 11246.1ms/step | 5.3 samples/s
|
| 485 |
+
Step 2770/50462 | Loss: 3.1259 | LR: 4.98e-04 | 7389.8ms/step | 8.1 samples/s
|
| 486 |
+
Step 2780/50462 | Loss: 3.1441 | LR: 4.98e-04 | 7386.7ms/step | 8.1 samples/s
|
| 487 |
+
Step 2790/50462 | Loss: 3.1499 | LR: 4.98e-04 | 7361.5ms/step | 8.2 samples/s
|
| 488 |
+
Step 2800/50462 | Loss: 3.1231 | LR: 4.98e-04 | 7362.2ms/step | 8.1 samples/s
|
| 489 |
+
Step 2810/50462 | Loss: 3.1289 | LR: 4.98e-04 | 7354.5ms/step | 8.2 samples/s
|
| 490 |
+
Step 2820/50462 | Loss: 3.1178 | LR: 4.98e-04 | 7373.0ms/step | 8.1 samples/s
|
| 491 |
+
Step 2830/50462 | Loss: 3.0993 | LR: 4.98e-04 | 7355.6ms/step | 8.2 samples/s
|
| 492 |
+
Step 2840/50462 | Loss: 3.1170 | LR: 4.98e-04 | 7344.4ms/step | 8.2 samples/s
|
| 493 |
+
Step 2850/50462 | Loss: 3.1312 | LR: 4.98e-04 | 7373.1ms/step | 8.1 samples/s
|
| 494 |
+
Step 2860/50462 | Loss: 3.1184 | LR: 4.98e-04 | 7376.8ms/step | 8.1 samples/s
|
| 495 |
+
Step 2870/50462 | Loss: 3.1153 | LR: 4.98e-04 | 7374.4ms/step | 8.1 samples/s
|
| 496 |
+
Step 2880/50462 | Loss: 3.1072 | LR: 4.98e-04 | 7364.4ms/step | 8.1 samples/s
|
| 497 |
+
Step 2890/50462 | Loss: 3.1181 | LR: 4.98e-04 | 7380.4ms/step | 8.1 samples/s
|
| 498 |
+
Step 2900/50462 | Loss: 3.1301 | LR: 4.98e-04 | 7360.8ms/step | 8.2 samples/s
|
| 499 |
+
Step 2910/50462 | Loss: 3.0896 | LR: 4.98e-04 | 7358.5ms/step | 8.2 samples/s
|
| 500 |
+
Step 2920/50462 | Loss: 3.1073 | LR: 4.98e-04 | 7381.8ms/step | 8.1 samples/s
|
| 501 |
+
Step 2930/50462 | Loss: 3.1197 | LR: 4.98e-04 | 7370.4ms/step | 8.1 samples/s
|
| 502 |
+
Step 2940/50462 | Loss: 3.0829 | LR: 4.98e-04 | 7377.3ms/step | 8.1 samples/s
|
| 503 |
+
Step 2950/50462 | Loss: 3.1117 | LR: 4.98e-04 | 7384.8ms/step | 8.1 samples/s
|
| 504 |
+
Step 2960/50462 | Loss: 3.1098 | LR: 4.98e-04 | 7358.4ms/step | 8.2 samples/s
|
| 505 |
+
Step 2970/50462 | Loss: 3.1180 | LR: 4.98e-04 | 7372.1ms/step | 8.1 samples/s
|
| 506 |
+
Step 2980/50462 | Loss: 3.1047 | LR: 4.98e-04 | 7366.9ms/step | 8.1 samples/s
|
| 507 |
+
Step 2990/50462 | Loss: 3.1069 | LR: 4.97e-04 | 7360.7ms/step | 8.2 samples/s
|
| 508 |
+
Step 3000/50462 | Loss: 3.0927 | LR: 4.97e-04 | 7360.1ms/step | 8.2 samples/s
|
| 509 |
+
Validation at step 3000: Loss = 3.1122, Perplexity = 22.47
|
| 510 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-3000
|
| 511 |
+
Step 3010/50462 | Loss: 3.0770 | LR: 4.97e-04 | 12146.7ms/step | 4.9 samples/s
|
| 512 |
+
Step 3020/50462 | Loss: 3.0881 | LR: 4.97e-04 | 7369.0ms/step | 8.1 samples/s
|
| 513 |
+
Step 3030/50462 | Loss: 3.1111 | LR: 4.97e-04 | 7362.5ms/step | 8.1 samples/s
|
| 514 |
+
Step 3040/50462 | Loss: 3.1004 | LR: 4.97e-04 | 7355.5ms/step | 8.2 samples/s
|
| 515 |
+
Step 3050/50462 | Loss: 3.0841 | LR: 4.97e-04 | 7369.9ms/step | 8.1 samples/s
|
| 516 |
+
Step 3060/50462 | Loss: 3.1183 | LR: 4.97e-04 | 7351.9ms/step | 8.2 samples/s
|
| 517 |
+
Step 3070/50462 | Loss: 3.1183 | LR: 4.97e-04 | 7372.4ms/step | 8.1 samples/s
|
| 518 |
+
Step 3080/50462 | Loss: 3.0967 | LR: 4.97e-04 | 7378.5ms/step | 8.1 samples/s
|
| 519 |
+
Step 3090/50462 | Loss: 3.1079 | LR: 4.97e-04 | 7395.7ms/step | 8.1 samples/s
|
| 520 |
+
Step 3100/50462 | Loss: 3.1020 | LR: 4.97e-04 | 7380.6ms/step | 8.1 samples/s
|
| 521 |
+
Step 3110/50462 | Loss: 3.0934 | LR: 4.97e-04 | 7358.4ms/step | 8.2 samples/s
|
| 522 |
+
Step 3120/50462 | Loss: 3.0919 | LR: 4.97e-04 | 7363.8ms/step | 8.1 samples/s
|
| 523 |
+
Step 3130/50462 | Loss: 3.0860 | LR: 4.97e-04 | 7368.5ms/step | 8.1 samples/s
|
| 524 |
+
Step 3140/50462 | Loss: 3.1037 | LR: 4.97e-04 | 7366.7ms/step | 8.1 samples/s
|
| 525 |
+
Step 3150/50462 | Loss: 3.0955 | LR: 4.97e-04 | 7369.2ms/step | 8.1 samples/s
|
| 526 |
+
Step 3160/50462 | Loss: 3.0900 | LR: 4.97e-04 | 7368.1ms/step | 8.1 samples/s
|
| 527 |
+
Step 3170/50462 | Loss: 3.0800 | LR: 4.97e-04 | 7368.7ms/step | 8.1 samples/s
|
| 528 |
+
Step 3180/50462 | Loss: 3.0779 | LR: 4.97e-04 | 7364.3ms/step | 8.1 samples/s
|
| 529 |
+
Step 3190/50462 | Loss: 3.0759 | LR: 4.97e-04 | 7358.5ms/step | 8.2 samples/s
|
| 530 |
+
Step 3200/50462 | Loss: 3.0620 | LR: 4.97e-04 | 7349.9ms/step | 8.2 samples/s
|
| 531 |
+
Step 3210/50462 | Loss: 3.0839 | LR: 4.97e-04 | 7366.5ms/step | 8.1 samples/s
|
| 532 |
+
Step 3220/50462 | Loss: 3.0646 | LR: 4.97e-04 | 7389.4ms/step | 8.1 samples/s
|
| 533 |
+
Step 3230/50462 | Loss: 3.0610 | LR: 4.97e-04 | 7372.2ms/step | 8.1 samples/s
|
| 534 |
+
Step 3240/50462 | Loss: 3.0832 | LR: 4.97e-04 | 7349.6ms/step | 8.2 samples/s
|
| 535 |
+
Step 3250/50462 | Loss: 3.0886 | LR: 4.97e-04 | 7361.0ms/step | 8.2 samples/s
|
| 536 |
+
Validation at step 3250: Loss = 3.0823, Perplexity = 21.81
|
| 537 |
+
Step 3260/50462 | Loss: 3.0558 | LR: 4.97e-04 | 11233.7ms/step | 5.3 samples/s
|
| 538 |
+
Step 3270/50462 | Loss: 3.0566 | LR: 4.97e-04 | 7398.0ms/step | 8.1 samples/s
|
| 539 |
+
Step 3280/50462 | Loss: 3.0632 | LR: 4.97e-04 | 7358.3ms/step | 8.2 samples/s
|
| 540 |
+
Step 3290/50462 | Loss: 3.0667 | LR: 4.97e-04 | 7353.2ms/step | 8.2 samples/s
|
| 541 |
+
Step 3300/50462 | Loss: 3.0615 | LR: 4.97e-04 | 7352.4ms/step | 8.2 samples/s
|
| 542 |
+
Step 3310/50462 | Loss: 3.0545 | LR: 4.97e-04 | 7351.0ms/step | 8.2 samples/s
|
| 543 |
+
Step 3320/50462 | Loss: 3.0483 | LR: 4.97e-04 | 7390.7ms/step | 8.1 samples/s
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch.log
ADDED
|
@@ -0,0 +1,1024 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-16750",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": true,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b mixed 8k seed70 6gpu resume",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"mixed",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": null,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 35,
|
| 58 |
+
"sparse_mask_pool_size": 10000,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 62 |
+
"sparse_mask_seed": 70,
|
| 63 |
+
"sparse_mixed_nrb_set": [
|
| 64 |
+
63,
|
| 65 |
+
47,
|
| 66 |
+
31,
|
| 67 |
+
21,
|
| 68 |
+
11
|
| 69 |
+
],
|
| 70 |
+
"dense_mode": false,
|
| 71 |
+
"dense_step_intervals": null
|
| 72 |
+
}
|
| 73 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 74 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-16750
|
| 75 |
+
Checkpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: TrueCheckpoint activations: True
|
| 76 |
+
Checkpoint activations: True
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
Checkpoint activations: True
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
================================================================================
|
| 84 |
+
MODEL ARCHITECTURE
|
| 85 |
+
================================================================================
|
| 86 |
+
Model config:
|
| 87 |
+
vocab_size: 151936
|
| 88 |
+
hidden_size: 1536
|
| 89 |
+
num_hidden_layers: 28
|
| 90 |
+
num_attention_heads: 12
|
| 91 |
+
num_key_value_heads: 4
|
| 92 |
+
head_dim: 128
|
| 93 |
+
intermediate_size: 6144
|
| 94 |
+
max_position_embeddings: 32768
|
| 95 |
+
rope_theta: 1000000
|
| 96 |
+
rope_train_position_scale_set: None
|
| 97 |
+
attention_type: sparse_stochastic
|
| 98 |
+
sparse_block_size: 128
|
| 99 |
+
sparse_num_random_blocks: 35
|
| 100 |
+
sparse_mask_pool_size: 10000
|
| 101 |
+
sparse_max_seq_length: 8192
|
| 102 |
+
|
| 103 |
+
Parameter count:
|
| 104 |
+
Total parameters: 1,202,352,640
|
| 105 |
+
Trainable parameters: 1,202,352,640
|
| 106 |
+
Non-trainable parameters: 0
|
| 107 |
+
Model size (BF16): 2.40 GB
|
| 108 |
+
|
| 109 |
+
--------------------------------------------------------------------------------
|
| 110 |
+
LAYER-BY-LAYER SUMMARY
|
| 111 |
+
--------------------------------------------------------------------------------
|
| 112 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 113 |
+
|
| 114 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 115 |
+
|
| 116 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 117 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 118 |
+
|
| 119 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 120 |
+
_warn_once(
|
| 121 |
+
================================================================================================================================================================
|
| 122 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 123 |
+
================================================================================================================================================================
|
| 124 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 125 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 126 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 127 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 153 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 154 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 155 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 156 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 157 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 158 |
+
================================================================================================================================================================
|
| 159 |
+
Total params: 1,435,726,336
|
| 160 |
+
Trainable params: 1,435,726,336
|
| 161 |
+
Non-trainable params: 0
|
| 162 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 163 |
+
================================================================================================================================================================
|
| 164 |
+
Input size (MB): 0.07
|
| 165 |
+
Forward/backward pass size (MB): 26218.59
|
| 166 |
+
Params size (MB): 2871.45
|
| 167 |
+
Estimated Total Size (MB): 29090.11
|
| 168 |
+
================================================================================================================================================================
|
| 169 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 170 |
+
================================================================================
|
| 171 |
+
|
| 172 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 173 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 174 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 175 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 176 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 177 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 178 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 179 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 180 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 184 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 185 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 186 |
+
Dataset split: 3027764 train, 159356 validation
|
| 187 |
+
Training examples (packed, train split): 3027764
|
| 188 |
+
Validation examples (packed, val split): 159356
|
| 189 |
+
Per-device batch size: 1
|
| 190 |
+
World size (GPUs): 6
|
| 191 |
+
Gradient accumulation steps: 10
|
| 192 |
+
Effective batch size (seqs/step): 60
|
| 193 |
+
Sequence length: 8192
|
| 194 |
+
Tokens per optimizer step: 491,520
|
| 195 |
+
Micro-batches per epoch (per rank): 504628
|
| 196 |
+
Optimizer steps per epoch: 50462
|
| 197 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 198 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-16750...
|
| 199 |
+
Preserving checkpoint LR state for exact resume: current=[0.00038272506474164547], schedule_base=[0.0005]
|
| 200 |
+
Resumed from checkpoint at step 16750 (epoch 0, micro_step 182500)
|
| 201 |
+
Resuming training from step 16750
|
| 202 |
+
Resuming wandb run i8ewyy95
|
| 203 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 204 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 205 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 206 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260823_150341-i8ewyy95
|
| 207 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 208 |
+
wandb: Resuming run 1.2b mixed 8k seed70 6gpu resume
|
| 209 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 210 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i8ewyy95
|
| 211 |
+
Starting training...
|
| 212 |
+
Validation at step 16750: Loss = 2.6770, Perplexity = 14.54
|
| 213 |
+
wandb: WARNING Tried to log to step 16750 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 214 |
+
Switching attention mode for step 16751: sparse_stochastic (configured sparse attention)
|
| 215 |
+
Step 16760/50462 | Loss: 2.6845 | LR: 3.83e-04 | 42321.5ms/step | 1.4 samples/s
|
| 216 |
+
wandb: WARNING Tried to log to step 16760 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 217 |
+
Step 16770/50462 | Loss: 2.6890 | LR: 3.82e-04 | 2741.6ms/step | 21.9 samples/s
|
| 218 |
+
wandb: WARNING Tried to log to step 16770 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 219 |
+
Step 16780/50462 | Loss: 2.6879 | LR: 3.82e-04 | 2748.5ms/step | 21.8 samples/s
|
| 220 |
+
wandb: WARNING Tried to log to step 16780 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 221 |
+
Step 16790/50462 | Loss: 2.6737 | LR: 3.82e-04 | 2739.1ms/step | 21.9 samples/s
|
| 222 |
+
wandb: WARNING Tried to log to step 16790 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 223 |
+
Step 16800/50462 | Loss: 2.6492 | LR: 3.82e-04 | 2736.2ms/step | 21.9 samples/s
|
| 224 |
+
wandb: WARNING Tried to log to step 16800 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 225 |
+
Step 16810/50462 | Loss: 2.6735 | LR: 3.82e-04 | 2744.5ms/step | 21.9 samples/s
|
| 226 |
+
wandb: WARNING Tried to log to step 16810 that is less than the current step 16811. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 227 |
+
Step 16820/50462 | Loss: 2.6886 | LR: 3.82e-04 | 2740.7ms/step | 21.9 samples/s
|
| 228 |
+
Step 16830/50462 | Loss: 2.6647 | LR: 3.82e-04 | 2738.4ms/step | 21.9 samples/s
|
| 229 |
+
Step 16840/50462 | Loss: 2.6860 | LR: 3.82e-04 | 2741.9ms/step | 21.9 samples/s
|
| 230 |
+
Step 16850/50462 | Loss: 2.6593 | LR: 3.81e-04 | 2734.1ms/step | 21.9 samples/s
|
| 231 |
+
Step 16860/50462 | Loss: 2.6914 | LR: 3.81e-04 | 2751.8ms/step | 21.8 samples/s
|
| 232 |
+
Step 16870/50462 | Loss: 2.7246 | LR: 3.81e-04 | 2746.3ms/step | 21.8 samples/s
|
| 233 |
+
Step 16880/50462 | Loss: 2.6777 | LR: 3.81e-04 | 2743.7ms/step | 21.9 samples/s
|
| 234 |
+
Step 16890/50462 | Loss: 2.6550 | LR: 3.81e-04 | 2740.4ms/step | 21.9 samples/s
|
| 235 |
+
Step 16900/50462 | Loss: 2.6746 | LR: 3.81e-04 | 2747.2ms/step | 21.8 samples/s
|
| 236 |
+
Step 16910/50462 | Loss: 2.7249 | LR: 3.81e-04 | 2734.0ms/step | 21.9 samples/s
|
| 237 |
+
Step 16920/50462 | Loss: 2.6715 | LR: 3.80e-04 | 2737.6ms/step | 21.9 samples/s
|
| 238 |
+
Step 16930/50462 | Loss: 2.6982 | LR: 3.80e-04 | 2739.3ms/step | 21.9 samples/s
|
| 239 |
+
Step 16940/50462 | Loss: 2.6699 | LR: 3.80e-04 | 2741.7ms/step | 21.9 samples/s
|
| 240 |
+
Step 16950/50462 | Loss: 2.6797 | LR: 3.80e-04 | 2745.6ms/step | 21.9 samples/s
|
| 241 |
+
Step 16960/50462 | Loss: 2.6754 | LR: 3.80e-04 | 2747.9ms/step | 21.8 samples/s
|
| 242 |
+
Step 16970/50462 | Loss: 2.6537 | LR: 3.80e-04 | 2748.1ms/step | 21.8 samples/s
|
| 243 |
+
Step 16980/50462 | Loss: 2.6712 | LR: 3.80e-04 | 2738.0ms/step | 21.9 samples/s
|
| 244 |
+
Step 16990/50462 | Loss: 2.6581 | LR: 3.79e-04 | 2754.7ms/step | 21.8 samples/s
|
| 245 |
+
Step 17000/50462 | Loss: 2.7017 | LR: 3.79e-04 | 2724.3ms/step | 22.0 samples/s
|
| 246 |
+
Validation at step 17000: Loss = 2.6745, Perplexity = 14.51
|
| 247 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17000
|
| 248 |
+
Step 17010/50462 | Loss: 2.6479 | LR: 3.79e-04 | 4095.8ms/step | 14.6 samples/s
|
| 249 |
+
Step 17020/50462 | Loss: 2.7154 | LR: 3.79e-04 | 2732.7ms/step | 22.0 samples/s
|
| 250 |
+
Step 17030/50462 | Loss: 2.6598 | LR: 3.79e-04 | 2744.0ms/step | 21.9 samples/s
|
| 251 |
+
Step 17040/50462 | Loss: 2.7000 | LR: 3.79e-04 | 2741.9ms/step | 21.9 samples/s
|
| 252 |
+
Step 17050/50462 | Loss: 2.7051 | LR: 3.79e-04 | 2742.7ms/step | 21.9 samples/s
|
| 253 |
+
Step 17060/50462 | Loss: 2.6419 | LR: 3.79e-04 | 2741.8ms/step | 21.9 samples/s
|
| 254 |
+
Step 17070/50462 | Loss: 2.6524 | LR: 3.78e-04 | 2744.3ms/step | 21.9 samples/s
|
| 255 |
+
Step 17080/50462 | Loss: 2.7199 | LR: 3.78e-04 | 2750.9ms/step | 21.8 samples/s
|
| 256 |
+
Step 17090/50462 | Loss: 2.6643 | LR: 3.78e-04 | 2733.8ms/step | 21.9 samples/s
|
| 257 |
+
Step 17100/50462 | Loss: 2.6472 | LR: 3.78e-04 | 2747.4ms/step | 21.8 samples/s
|
| 258 |
+
Step 17110/50462 | Loss: 2.6479 | LR: 3.78e-04 | 2728.8ms/step | 22.0 samples/s
|
| 259 |
+
Step 17120/50462 | Loss: 2.6519 | LR: 3.78e-04 | 2737.4ms/step | 21.9 samples/s
|
| 260 |
+
Step 17130/50462 | Loss: 2.7193 | LR: 3.78e-04 | 2741.4ms/step | 21.9 samples/s
|
| 261 |
+
Step 17140/50462 | Loss: 2.6912 | LR: 3.77e-04 | 2745.5ms/step | 21.9 samples/s
|
| 262 |
+
Step 17150/50462 | Loss: 2.6687 | LR: 3.77e-04 | 2743.4ms/step | 21.9 samples/s
|
| 263 |
+
Step 17160/50462 | Loss: 2.6548 | LR: 3.77e-04 | 2739.5ms/step | 21.9 samples/s
|
| 264 |
+
Step 17170/50462 | Loss: 2.6578 | LR: 3.77e-04 | 2745.8ms/step | 21.9 samples/s
|
| 265 |
+
Step 17180/50462 | Loss: 2.6645 | LR: 3.77e-04 | 2744.1ms/step | 21.9 samples/s
|
| 266 |
+
Step 17190/50462 | Loss: 2.6589 | LR: 3.77e-04 | 2742.1ms/step | 21.9 samples/s
|
| 267 |
+
Step 17200/50462 | Loss: 2.6407 | LR: 3.77e-04 | 2738.5ms/step | 21.9 samples/s
|
| 268 |
+
Step 17210/50462 | Loss: 2.6723 | LR: 3.77e-04 | 2736.7ms/step | 21.9 samples/s
|
| 269 |
+
Step 17220/50462 | Loss: 2.6787 | LR: 3.76e-04 | 2737.8ms/step | 21.9 samples/s
|
| 270 |
+
Step 17230/50462 | Loss: 2.6773 | LR: 3.76e-04 | 2748.0ms/step | 21.8 samples/s
|
| 271 |
+
Step 17240/50462 | Loss: 2.7250 | LR: 3.76e-04 | 2745.2ms/step | 21.9 samples/s
|
| 272 |
+
Step 17250/50462 | Loss: 2.6901 | LR: 3.76e-04 | 2740.2ms/step | 21.9 samples/s
|
| 273 |
+
Validation at step 17250: Loss = 2.6714, Perplexity = 14.46
|
| 274 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17250
|
| 275 |
+
Step 17260/50462 | Loss: 2.6524 | LR: 3.76e-04 | 4167.8ms/step | 14.4 samples/s
|
| 276 |
+
Step 17270/50462 | Loss: 2.6861 | LR: 3.76e-04 | 2739.5ms/step | 21.9 samples/s
|
| 277 |
+
Step 17280/50462 | Loss: 2.6878 | LR: 3.76e-04 | 2746.9ms/step | 21.8 samples/s
|
| 278 |
+
Step 17290/50462 | Loss: 2.6853 | LR: 3.75e-04 | 2733.2ms/step | 22.0 samples/s
|
| 279 |
+
Step 17300/50462 | Loss: 2.6508 | LR: 3.75e-04 | 2745.2ms/step | 21.9 samples/s
|
| 280 |
+
Step 17310/50462 | Loss: 2.6582 | LR: 3.75e-04 | 2740.2ms/step | 21.9 samples/s
|
| 281 |
+
Step 17320/50462 | Loss: 2.6454 | LR: 3.75e-04 | 2732.3ms/step | 22.0 samples/s
|
| 282 |
+
Step 17330/50462 | Loss: 2.7101 | LR: 3.75e-04 | 2735.3ms/step | 21.9 samples/s
|
| 283 |
+
Step 17340/50462 | Loss: 2.6979 | LR: 3.75e-04 | 2744.4ms/step | 21.9 samples/s
|
| 284 |
+
Step 17350/50462 | Loss: 2.7007 | LR: 3.75e-04 | 2739.7ms/step | 21.9 samples/s
|
| 285 |
+
Step 17360/50462 | Loss: 2.6736 | LR: 3.74e-04 | 2740.2ms/step | 21.9 samples/s
|
| 286 |
+
Step 17370/50462 | Loss: 2.6425 | LR: 3.74e-04 | 2736.9ms/step | 21.9 samples/s
|
| 287 |
+
Step 17380/50462 | Loss: 2.7250 | LR: 3.74e-04 | 2738.6ms/step | 21.9 samples/s
|
| 288 |
+
Step 17390/50462 | Loss: 2.7061 | LR: 3.74e-04 | 2744.2ms/step | 21.9 samples/s
|
| 289 |
+
Step 17400/50462 | Loss: 2.6572 | LR: 3.74e-04 | 2752.7ms/step | 21.8 samples/s
|
| 290 |
+
Step 17410/50462 | Loss: 2.6793 | LR: 3.74e-04 | 2731.4ms/step | 22.0 samples/s
|
| 291 |
+
Step 17420/50462 | Loss: 2.6298 | LR: 3.74e-04 | 2729.2ms/step | 22.0 samples/s
|
| 292 |
+
Step 17430/50462 | Loss: 2.6423 | LR: 3.74e-04 | 2735.2ms/step | 21.9 samples/s
|
| 293 |
+
Step 17440/50462 | Loss: 2.6059 | LR: 3.73e-04 | 2741.7ms/step | 21.9 samples/s
|
| 294 |
+
Step 17450/50462 | Loss: 2.6906 | LR: 3.73e-04 | 2738.0ms/step | 21.9 samples/s
|
| 295 |
+
Step 17460/50462 | Loss: 2.6816 | LR: 3.73e-04 | 2735.4ms/step | 21.9 samples/s
|
| 296 |
+
Step 17470/50462 | Loss: 2.6713 | LR: 3.73e-04 | 2733.6ms/step | 21.9 samples/s
|
| 297 |
+
Step 17480/50462 | Loss: 2.6729 | LR: 3.73e-04 | 2742.8ms/step | 21.9 samples/s
|
| 298 |
+
Step 17490/50462 | Loss: 2.7170 | LR: 3.73e-04 | 2733.3ms/step | 22.0 samples/s
|
| 299 |
+
Step 17500/50462 | Loss: 2.6628 | LR: 3.73e-04 | 2738.7ms/step | 21.9 samples/s
|
| 300 |
+
Validation at step 17500: Loss = 2.6724, Perplexity = 14.47
|
| 301 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17500
|
| 302 |
+
Step 17510/50462 | Loss: 2.6499 | LR: 3.72e-04 | 4209.1ms/step | 14.3 samples/s
|
| 303 |
+
Step 17520/50462 | Loss: 2.6490 | LR: 3.72e-04 | 2745.2ms/step | 21.9 samples/s
|
| 304 |
+
Step 17530/50462 | Loss: 2.6474 | LR: 3.72e-04 | 2733.7ms/step | 21.9 samples/s
|
| 305 |
+
Step 17540/50462 | Loss: 2.6584 | LR: 3.72e-04 | 2741.5ms/step | 21.9 samples/s
|
| 306 |
+
Step 17550/50462 | Loss: 2.6596 | LR: 3.72e-04 | 2747.0ms/step | 21.8 samples/s
|
| 307 |
+
Step 17560/50462 | Loss: 2.6277 | LR: 3.72e-04 | 2733.8ms/step | 21.9 samples/s
|
| 308 |
+
Step 17570/50462 | Loss: 2.6458 | LR: 3.72e-04 | 2731.4ms/step | 22.0 samples/s
|
| 309 |
+
Step 17580/50462 | Loss: 2.6486 | LR: 3.71e-04 | 2731.8ms/step | 22.0 samples/s
|
| 310 |
+
Step 17590/50462 | Loss: 2.6779 | LR: 3.71e-04 | 2735.2ms/step | 21.9 samples/s
|
| 311 |
+
Step 17600/50462 | Loss: 2.6830 | LR: 3.71e-04 | 2733.2ms/step | 22.0 samples/s
|
| 312 |
+
Step 17610/50462 | Loss: 2.7174 | LR: 3.71e-04 | 2732.7ms/step | 22.0 samples/s
|
| 313 |
+
Step 17620/50462 | Loss: 2.6315 | LR: 3.71e-04 | 2742.0ms/step | 21.9 samples/s
|
| 314 |
+
Step 17630/50462 | Loss: 2.7254 | LR: 3.71e-04 | 2733.7ms/step | 21.9 samples/s
|
| 315 |
+
Step 17640/50462 | Loss: 2.6422 | LR: 3.71e-04 | 2737.1ms/step | 21.9 samples/s
|
| 316 |
+
Step 17650/50462 | Loss: 2.6546 | LR: 3.70e-04 | 2736.9ms/step | 21.9 samples/s
|
| 317 |
+
Step 17660/50462 | Loss: 2.6475 | LR: 3.70e-04 | 2735.7ms/step | 21.9 samples/s
|
| 318 |
+
Step 17670/50462 | Loss: 2.6282 | LR: 3.70e-04 | 2746.5ms/step | 21.8 samples/s
|
| 319 |
+
Step 17680/50462 | Loss: 2.6354 | LR: 3.70e-04 | 2731.6ms/step | 22.0 samples/s
|
| 320 |
+
Step 17690/50462 | Loss: 2.6998 | LR: 3.70e-04 | 2736.1ms/step | 21.9 samples/s
|
| 321 |
+
Step 17700/50462 | Loss: 2.6663 | LR: 3.70e-04 | 2734.1ms/step | 21.9 samples/s
|
| 322 |
+
Step 17710/50462 | Loss: 2.6304 | LR: 3.70e-04 | 2739.0ms/step | 21.9 samples/s
|
| 323 |
+
Step 17720/50462 | Loss: 2.6309 | LR: 3.69e-04 | 2734.9ms/step | 21.9 samples/s
|
| 324 |
+
Step 17730/50462 | Loss: 2.6613 | LR: 3.69e-04 | 2736.2ms/step | 21.9 samples/s
|
| 325 |
+
Step 17740/50462 | Loss: 2.6881 | LR: 3.69e-04 | 2732.6ms/step | 22.0 samples/s
|
| 326 |
+
Step 17750/50462 | Loss: 2.6692 | LR: 3.69e-04 | 2738.2ms/step | 21.9 samples/s
|
| 327 |
+
Validation at step 17750: Loss = 2.6653, Perplexity = 14.37
|
| 328 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-17750
|
| 329 |
+
Step 17760/50462 | Loss: 2.6379 | LR: 3.69e-04 | 4110.4ms/step | 14.6 samples/s
|
| 330 |
+
Step 17770/50462 | Loss: 2.6693 | LR: 3.69e-04 | 2728.9ms/step | 22.0 samples/s
|
| 331 |
+
Step 17780/50462 | Loss: 2.6336 | LR: 3.69e-04 | 2729.8ms/step | 22.0 samples/s
|
| 332 |
+
Step 17790/50462 | Loss: 2.6621 | LR: 3.69e-04 | 2726.5ms/step | 22.0 samples/s
|
| 333 |
+
Step 17800/50462 | Loss: 2.6762 | LR: 3.68e-04 | 2735.8ms/step | 21.9 samples/s
|
| 334 |
+
Step 17810/50462 | Loss: 2.6745 | LR: 3.68e-04 | 2736.1ms/step | 21.9 samples/s
|
| 335 |
+
Step 17820/50462 | Loss: 2.6650 | LR: 3.68e-04 | 2736.9ms/step | 21.9 samples/s
|
| 336 |
+
Step 17830/50462 | Loss: 2.6496 | LR: 3.68e-04 | 2739.1ms/step | 21.9 samples/s
|
| 337 |
+
Step 17840/50462 | Loss: 2.7096 | LR: 3.68e-04 | 2735.9ms/step | 21.9 samples/s
|
| 338 |
+
Step 17850/50462 | Loss: 2.6736 | LR: 3.68e-04 | 2736.1ms/step | 21.9 samples/s
|
| 339 |
+
Step 17860/50462 | Loss: 2.6645 | LR: 3.68e-04 | 2750.8ms/step | 21.8 samples/s
|
| 340 |
+
Step 17870/50462 | Loss: 2.7052 | LR: 3.67e-04 | 2742.4ms/step | 21.9 samples/s
|
| 341 |
+
Step 17880/50462 | Loss: 2.6519 | LR: 3.67e-04 | 2735.3ms/step | 21.9 samples/s
|
| 342 |
+
Step 17890/50462 | Loss: 2.6666 | LR: 3.67e-04 | 2732.2ms/step | 22.0 samples/s
|
| 343 |
+
Step 17900/50462 | Loss: 2.6343 | LR: 3.67e-04 | 2742.8ms/step | 21.9 samples/s
|
| 344 |
+
Step 17910/50462 | Loss: 2.6359 | LR: 3.67e-04 | 2736.5ms/step | 21.9 samples/s
|
| 345 |
+
Step 17920/50462 | Loss: 2.6724 | LR: 3.67e-04 | 2743.3ms/step | 21.9 samples/s
|
| 346 |
+
Step 17930/50462 | Loss: 2.6395 | LR: 3.67e-04 | 2729.1ms/step | 22.0 samples/s
|
| 347 |
+
Step 17940/50462 | Loss: 2.6774 | LR: 3.66e-04 | 2744.6ms/step | 21.9 samples/s
|
| 348 |
+
Step 17950/50462 | Loss: 2.6698 | LR: 3.66e-04 | 2737.7ms/step | 21.9 samples/s
|
| 349 |
+
Step 17960/50462 | Loss: 2.6940 | LR: 3.66e-04 | 2738.6ms/step | 21.9 samples/s
|
| 350 |
+
Step 17970/50462 | Loss: 2.6282 | LR: 3.66e-04 | 2738.9ms/step | 21.9 samples/s
|
| 351 |
+
Step 17980/50462 | Loss: 2.6692 | LR: 3.66e-04 | 2734.5ms/step | 21.9 samples/s
|
| 352 |
+
Step 17990/50462 | Loss: 2.6628 | LR: 3.66e-04 | 2729.8ms/step | 22.0 samples/s
|
| 353 |
+
Step 18000/50462 | Loss: 2.6642 | LR: 3.66e-04 | 2731.5ms/step | 22.0 samples/s
|
| 354 |
+
Validation at step 18000: Loss = 2.6625, Perplexity = 14.33
|
| 355 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18000
|
| 356 |
+
Step 18010/50462 | Loss: 2.6936 | LR: 3.65e-04 | 4074.4ms/step | 14.7 samples/s
|
| 357 |
+
Step 18020/50462 | Loss: 2.6692 | LR: 3.65e-04 | 2730.7ms/step | 22.0 samples/s
|
| 358 |
+
Step 18030/50462 | Loss: 2.6487 | LR: 3.65e-04 | 2737.9ms/step | 21.9 samples/s
|
| 359 |
+
Step 18040/50462 | Loss: 2.6453 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 360 |
+
Step 18050/50462 | Loss: 2.6541 | LR: 3.65e-04 | 2733.8ms/step | 21.9 samples/s
|
| 361 |
+
Step 18060/50462 | Loss: 2.6618 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 362 |
+
Step 18070/50462 | Loss: 2.7331 | LR: 3.65e-04 | 2734.5ms/step | 21.9 samples/s
|
| 363 |
+
Step 18080/50462 | Loss: 2.6736 | LR: 3.64e-04 | 2735.7ms/step | 21.9 samples/s
|
| 364 |
+
Step 18090/50462 | Loss: 2.6733 | LR: 3.64e-04 | 2739.5ms/step | 21.9 samples/s
|
| 365 |
+
Step 18100/50462 | Loss: 2.6680 | LR: 3.64e-04 | 2733.1ms/step | 22.0 samples/s
|
| 366 |
+
Step 18110/50462 | Loss: 2.6850 | LR: 3.64e-04 | 2740.1ms/step | 21.9 samples/s
|
| 367 |
+
Step 18120/50462 | Loss: 2.6658 | LR: 3.64e-04 | 2737.0ms/step | 21.9 samples/s
|
| 368 |
+
Step 18130/50462 | Loss: 2.6800 | LR: 3.64e-04 | 2732.1ms/step | 22.0 samples/s
|
| 369 |
+
Step 18140/50462 | Loss: 2.6739 | LR: 3.64e-04 | 2726.7ms/step | 22.0 samples/s
|
| 370 |
+
Step 18150/50462 | Loss: 2.6608 | LR: 3.63e-04 | 2730.1ms/step | 22.0 samples/s
|
| 371 |
+
Step 18160/50462 | Loss: 2.6525 | LR: 3.63e-04 | 2727.5ms/step | 22.0 samples/s
|
| 372 |
+
Step 18170/50462 | Loss: 2.6415 | LR: 3.63e-04 | 2727.3ms/step | 22.0 samples/s
|
| 373 |
+
Step 18180/50462 | Loss: 2.6567 | LR: 3.63e-04 | 2729.1ms/step | 22.0 samples/s
|
| 374 |
+
Step 18190/50462 | Loss: 2.6893 | LR: 3.63e-04 | 2735.0ms/step | 21.9 samples/s
|
| 375 |
+
Step 18200/50462 | Loss: 2.6453 | LR: 3.63e-04 | 2731.5ms/step | 22.0 samples/s
|
| 376 |
+
Step 18210/50462 | Loss: 2.6536 | LR: 3.63e-04 | 2731.3ms/step | 22.0 samples/s
|
| 377 |
+
Step 18220/50462 | Loss: 2.6949 | LR: 3.62e-04 | 2728.5ms/step | 22.0 samples/s
|
| 378 |
+
Step 18230/50462 | Loss: 2.6505 | LR: 3.62e-04 | 2728.7ms/step | 22.0 samples/s
|
| 379 |
+
Step 18240/50462 | Loss: 2.7005 | LR: 3.62e-04 | 2738.0ms/step | 21.9 samples/s
|
| 380 |
+
Step 18250/50462 | Loss: 2.6361 | LR: 3.62e-04 | 2730.0ms/step | 22.0 samples/s
|
| 381 |
+
Validation at step 18250: Loss = 2.6568, Perplexity = 14.25
|
| 382 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18250
|
| 383 |
+
Step 18260/50462 | Loss: 2.6532 | LR: 3.62e-04 | 4068.8ms/step | 14.7 samples/s
|
| 384 |
+
Step 18270/50462 | Loss: 2.6540 | LR: 3.62e-04 | 2725.4ms/step | 22.0 samples/s
|
| 385 |
+
Step 18280/50462 | Loss: 2.6575 | LR: 3.62e-04 | 2728.6ms/step | 22.0 samples/s
|
| 386 |
+
Step 18290/50462 | Loss: 2.6261 | LR: 3.62e-04 | 2726.9ms/step | 22.0 samples/s
|
| 387 |
+
Step 18300/50462 | Loss: 2.6784 | LR: 3.61e-04 | 2733.5ms/step | 22.0 samples/s
|
| 388 |
+
Step 18310/50462 | Loss: 2.6237 | LR: 3.61e-04 | 2732.6ms/step | 22.0 samples/s
|
| 389 |
+
Step 18320/50462 | Loss: 2.6791 | LR: 3.61e-04 | 2739.1ms/step | 21.9 samples/s
|
| 390 |
+
Step 18330/50462 | Loss: 2.6377 | LR: 3.61e-04 | 2738.0ms/step | 21.9 samples/s
|
| 391 |
+
Step 18340/50462 | Loss: 2.6544 | LR: 3.61e-04 | 2721.9ms/step | 22.0 samples/s
|
| 392 |
+
Step 18350/50462 | Loss: 2.6899 | LR: 3.61e-04 | 2735.6ms/step | 21.9 samples/s
|
| 393 |
+
Step 18360/50462 | Loss: 2.6950 | LR: 3.61e-04 | 2738.1ms/step | 21.9 samples/s
|
| 394 |
+
Step 18370/50462 | Loss: 2.6819 | LR: 3.60e-04 | 2731.8ms/step | 22.0 samples/s
|
| 395 |
+
Step 18380/50462 | Loss: 2.6574 | LR: 3.60e-04 | 2736.8ms/step | 21.9 samples/s
|
| 396 |
+
Step 18390/50462 | Loss: 2.6187 | LR: 3.60e-04 | 2730.4ms/step | 22.0 samples/s
|
| 397 |
+
Step 18400/50462 | Loss: 2.6540 | LR: 3.60e-04 | 2734.9ms/step | 21.9 samples/s
|
| 398 |
+
Step 18410/50462 | Loss: 2.6629 | LR: 3.60e-04 | 2728.9ms/step | 22.0 samples/s
|
| 399 |
+
Step 18420/50462 | Loss: 2.6746 | LR: 3.60e-04 | 2729.4ms/step | 22.0 samples/s
|
| 400 |
+
Step 18430/50462 | Loss: 2.6708 | LR: 3.60e-04 | 2730.2ms/step | 22.0 samples/s
|
| 401 |
+
Step 18440/50462 | Loss: 2.6403 | LR: 3.59e-04 | 2731.1ms/step | 22.0 samples/s
|
| 402 |
+
Step 18450/50462 | Loss: 2.6549 | LR: 3.59e-04 | 2732.7ms/step | 22.0 samples/s
|
| 403 |
+
Step 18460/50462 | Loss: 2.6867 | LR: 3.59e-04 | 2734.0ms/step | 21.9 samples/s
|
| 404 |
+
Step 18470/50462 | Loss: 2.6399 | LR: 3.59e-04 | 2734.2ms/step | 21.9 samples/s
|
| 405 |
+
Step 18480/50462 | Loss: 2.6712 | LR: 3.59e-04 | 2729.2ms/step | 22.0 samples/s
|
| 406 |
+
Step 18490/50462 | Loss: 2.6583 | LR: 3.59e-04 | 2743.3ms/step | 21.9 samples/s
|
| 407 |
+
Step 18500/50462 | Loss: 2.6141 | LR: 3.59e-04 | 2730.3ms/step | 22.0 samples/s
|
| 408 |
+
Validation at step 18500: Loss = 2.6571, Perplexity = 14.25
|
| 409 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18500
|
| 410 |
+
Step 18510/50462 | Loss: 2.6709 | LR: 3.58e-04 | 4073.5ms/step | 14.7 samples/s
|
| 411 |
+
Step 18520/50462 | Loss: 2.6600 | LR: 3.58e-04 | 2727.3ms/step | 22.0 samples/s
|
| 412 |
+
Step 18530/50462 | Loss: 2.6412 | LR: 3.58e-04 | 2730.3ms/step | 22.0 samples/s
|
| 413 |
+
Step 18540/50462 | Loss: 2.6809 | LR: 3.58e-04 | 2737.8ms/step | 21.9 samples/s
|
| 414 |
+
Step 18550/50462 | Loss: 2.6322 | LR: 3.58e-04 | 2733.1ms/step | 22.0 samples/s
|
| 415 |
+
Step 18560/50462 | Loss: 2.6070 | LR: 3.58e-04 | 2732.7ms/step | 22.0 samples/s
|
| 416 |
+
Step 18570/50462 | Loss: 2.6590 | LR: 3.58e-04 | 2731.0ms/step | 22.0 samples/s
|
| 417 |
+
Step 18580/50462 | Loss: 2.6038 | LR: 3.57e-04 | 2731.8ms/step | 22.0 samples/s
|
| 418 |
+
Step 18590/50462 | Loss: 2.6436 | LR: 3.57e-04 | 2734.1ms/step | 21.9 samples/s
|
| 419 |
+
Step 18600/50462 | Loss: 2.6454 | LR: 3.57e-04 | 2733.4ms/step | 22.0 samples/s
|
| 420 |
+
Step 18610/50462 | Loss: 2.6729 | LR: 3.57e-04 | 2734.3ms/step | 21.9 samples/s
|
| 421 |
+
Step 18620/50462 | Loss: 2.6297 | LR: 3.57e-04 | 2729.1ms/step | 22.0 samples/s
|
| 422 |
+
Step 18630/50462 | Loss: 2.6217 | LR: 3.57e-04 | 2732.8ms/step | 22.0 samples/s
|
| 423 |
+
Step 18640/50462 | Loss: 2.6318 | LR: 3.57e-04 | 2730.3ms/step | 22.0 samples/s
|
| 424 |
+
Step 18650/50462 | Loss: 2.6130 | LR: 3.56e-04 | 2727.3ms/step | 22.0 samples/s
|
| 425 |
+
Step 18660/50462 | Loss: 2.6246 | LR: 3.56e-04 | 2746.8ms/step | 21.8 samples/s
|
| 426 |
+
Step 18670/50462 | Loss: 2.6687 | LR: 3.56e-04 | 2741.2ms/step | 21.9 samples/s
|
| 427 |
+
Step 18680/50462 | Loss: 2.6482 | LR: 3.56e-04 | 2740.6ms/step | 21.9 samples/s
|
| 428 |
+
Step 18690/50462 | Loss: 2.6537 | LR: 3.56e-04 | 2742.4ms/step | 21.9 samples/s
|
| 429 |
+
Step 18700/50462 | Loss: 2.6520 | LR: 3.56e-04 | 2742.5ms/step | 21.9 samples/s
|
| 430 |
+
Step 18710/50462 | Loss: 2.6375 | LR: 3.56e-04 | 2741.4ms/step | 21.9 samples/s
|
| 431 |
+
Step 18720/50462 | Loss: 2.6420 | LR: 3.55e-04 | 2735.4ms/step | 21.9 samples/s
|
| 432 |
+
Step 18730/50462 | Loss: 2.6709 | LR: 3.55e-04 | 2723.8ms/step | 22.0 samples/s
|
| 433 |
+
Step 18740/50462 | Loss: 2.6446 | LR: 3.55e-04 | 2746.4ms/step | 21.8 samples/s
|
| 434 |
+
Step 18750/50462 | Loss: 2.6592 | LR: 3.55e-04 | 2746.0ms/step | 21.8 samples/s
|
| 435 |
+
Validation at step 18750: Loss = 2.6571, Perplexity = 14.26
|
| 436 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-18750
|
| 437 |
+
Step 18760/50462 | Loss: 2.6690 | LR: 3.55e-04 | 4123.2ms/step | 14.6 samples/s
|
| 438 |
+
Step 18770/50462 | Loss: 2.6084 | LR: 3.55e-04 | 2746.6ms/step | 21.8 samples/s
|
| 439 |
+
Step 18780/50462 | Loss: 2.6240 | LR: 3.55e-04 | 2734.4ms/step | 21.9 samples/s
|
| 440 |
+
Step 18790/50462 | Loss: 2.7283 | LR: 3.54e-04 | 2724.6ms/step | 22.0 samples/s
|
| 441 |
+
Step 18800/50462 | Loss: 2.6264 | LR: 3.54e-04 | 2734.5ms/step | 21.9 samples/s
|
| 442 |
+
Step 18810/50462 | Loss: 2.6524 | LR: 3.54e-04 | 2736.2ms/step | 21.9 samples/s
|
| 443 |
+
Step 18820/50462 | Loss: 2.6710 | LR: 3.54e-04 | 2727.1ms/step | 22.0 samples/s
|
| 444 |
+
Step 18830/50462 | Loss: 2.6730 | LR: 3.54e-04 | 2725.1ms/step | 22.0 samples/s
|
| 445 |
+
Step 18840/50462 | Loss: 2.6750 | LR: 3.54e-04 | 2727.7ms/step | 22.0 samples/s
|
| 446 |
+
Step 18850/50462 | Loss: 2.6786 | LR: 3.54e-04 | 2733.9ms/step | 21.9 samples/s
|
| 447 |
+
Step 18860/50462 | Loss: 2.6556 | LR: 3.53e-04 | 2733.4ms/step | 22.0 samples/s
|
| 448 |
+
Step 18870/50462 | Loss: 2.7275 | LR: 3.53e-04 | 2725.6ms/step | 22.0 samples/s
|
| 449 |
+
Step 18880/50462 | Loss: 2.6388 | LR: 3.53e-04 | 2732.3ms/step | 22.0 samples/s
|
| 450 |
+
Step 18890/50462 | Loss: 2.6753 | LR: 3.53e-04 | 2734.6ms/step | 21.9 samples/s
|
| 451 |
+
Step 18900/50462 | Loss: 2.6620 | LR: 3.53e-04 | 2731.9ms/step | 22.0 samples/s
|
| 452 |
+
Step 18910/50462 | Loss: 2.6348 | LR: 3.53e-04 | 2732.1ms/step | 22.0 samples/s
|
| 453 |
+
Step 18920/50462 | Loss: 2.6709 | LR: 3.53e-04 | 2728.0ms/step | 22.0 samples/s
|
| 454 |
+
Step 18930/50462 | Loss: 2.6344 | LR: 3.52e-04 | 2719.1ms/step | 22.1 samples/s
|
| 455 |
+
Step 18940/50462 | Loss: 2.6501 | LR: 3.52e-04 | 2723.8ms/step | 22.0 samples/s
|
| 456 |
+
Step 18950/50462 | Loss: 2.6530 | LR: 3.52e-04 | 2725.3ms/step | 22.0 samples/s
|
| 457 |
+
Step 18960/50462 | Loss: 2.6482 | LR: 3.52e-04 | 2730.9ms/step | 22.0 samples/s
|
| 458 |
+
Step 18970/50462 | Loss: 2.6611 | LR: 3.52e-04 | 2720.8ms/step | 22.1 samples/s
|
| 459 |
+
Step 18980/50462 | Loss: 2.6298 | LR: 3.52e-04 | 2734.1ms/step | 21.9 samples/s
|
| 460 |
+
Step 18990/50462 | Loss: 2.6984 | LR: 3.52e-04 | 2741.2ms/step | 21.9 samples/s
|
| 461 |
+
Step 19000/50462 | Loss: 2.6279 | LR: 3.51e-04 | 2743.1ms/step | 21.9 samples/s
|
| 462 |
+
Validation at step 19000: Loss = 2.6523, Perplexity = 14.19
|
| 463 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19000
|
| 464 |
+
Step 19010/50462 | Loss: 2.6076 | LR: 3.51e-04 | 4144.8ms/step | 14.5 samples/s
|
| 465 |
+
Step 19020/50462 | Loss: 2.6465 | LR: 3.51e-04 | 2733.1ms/step | 22.0 samples/s
|
| 466 |
+
Step 19030/50462 | Loss: 2.6569 | LR: 3.51e-04 | 2734.2ms/step | 21.9 samples/s
|
| 467 |
+
Step 19040/50462 | Loss: 2.6762 | LR: 3.51e-04 | 2737.5ms/step | 21.9 samples/s
|
| 468 |
+
Step 19050/50462 | Loss: 2.6975 | LR: 3.51e-04 | 2735.8ms/step | 21.9 samples/s
|
| 469 |
+
Step 19060/50462 | Loss: 2.6794 | LR: 3.50e-04 | 2743.4ms/step | 21.9 samples/s
|
| 470 |
+
Step 19070/50462 | Loss: 2.6772 | LR: 3.50e-04 | 2725.8ms/step | 22.0 samples/s
|
| 471 |
+
Step 19080/50462 | Loss: 2.6643 | LR: 3.50e-04 | 2726.9ms/step | 22.0 samples/s
|
| 472 |
+
Step 19090/50462 | Loss: 2.6438 | LR: 3.50e-04 | 2734.7ms/step | 21.9 samples/s
|
| 473 |
+
Step 19100/50462 | Loss: 2.6369 | LR: 3.50e-04 | 2726.4ms/step | 22.0 samples/s
|
| 474 |
+
Step 19110/50462 | Loss: 2.6664 | LR: 3.50e-04 | 2731.8ms/step | 22.0 samples/s
|
| 475 |
+
Step 19120/50462 | Loss: 2.6536 | LR: 3.50e-04 | 2730.6ms/step | 22.0 samples/s
|
| 476 |
+
Step 19130/50462 | Loss: 2.6321 | LR: 3.49e-04 | 2724.8ms/step | 22.0 samples/s
|
| 477 |
+
Step 19140/50462 | Loss: 2.6259 | LR: 3.49e-04 | 2732.3ms/step | 22.0 samples/s
|
| 478 |
+
Step 19150/50462 | Loss: 2.6396 | LR: 3.49e-04 | 2728.0ms/step | 22.0 samples/s
|
| 479 |
+
Step 19160/50462 | Loss: 2.6879 | LR: 3.49e-04 | 2732.7ms/step | 22.0 samples/s
|
| 480 |
+
Step 19170/50462 | Loss: 2.6140 | LR: 3.49e-04 | 2744.2ms/step | 21.9 samples/s
|
| 481 |
+
Step 19180/50462 | Loss: 2.6137 | LR: 3.49e-04 | 2735.8ms/step | 21.9 samples/s
|
| 482 |
+
Step 19190/50462 | Loss: 2.6189 | LR: 3.49e-04 | 2736.9ms/step | 21.9 samples/s
|
| 483 |
+
Step 19200/50462 | Loss: 2.6316 | LR: 3.48e-04 | 2732.4ms/step | 22.0 samples/s
|
| 484 |
+
Step 19210/50462 | Loss: 2.6419 | LR: 3.48e-04 | 2730.9ms/step | 22.0 samples/s
|
| 485 |
+
Step 19220/50462 | Loss: 2.6276 | LR: 3.48e-04 | 2728.5ms/step | 22.0 samples/s
|
| 486 |
+
Step 19230/50462 | Loss: 2.6202 | LR: 3.48e-04 | 2734.4ms/step | 21.9 samples/s
|
| 487 |
+
Step 19240/50462 | Loss: 2.6284 | LR: 3.48e-04 | 2732.9ms/step | 22.0 samples/s
|
| 488 |
+
Step 19250/50462 | Loss: 2.6117 | LR: 3.48e-04 | 2739.4ms/step | 21.9 samples/s
|
| 489 |
+
Validation at step 19250: Loss = 2.6510, Perplexity = 14.17
|
| 490 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19250
|
| 491 |
+
Step 19260/50462 | Loss: 2.6737 | LR: 3.48e-04 | 4093.0ms/step | 14.7 samples/s
|
| 492 |
+
Step 19270/50462 | Loss: 2.6222 | LR: 3.47e-04 | 2733.5ms/step | 21.9 samples/s
|
| 493 |
+
Step 19280/50462 | Loss: 2.6719 | LR: 3.47e-04 | 2726.4ms/step | 22.0 samples/s
|
| 494 |
+
Step 19290/50462 | Loss: 2.6361 | LR: 3.47e-04 | 2728.0ms/step | 22.0 samples/s
|
| 495 |
+
Step 19300/50462 | Loss: 2.6214 | LR: 3.47e-04 | 2740.0ms/step | 21.9 samples/s
|
| 496 |
+
Step 19310/50462 | Loss: 2.6255 | LR: 3.47e-04 | 2736.7ms/step | 21.9 samples/s
|
| 497 |
+
Step 19320/50462 | Loss: 2.6366 | LR: 3.47e-04 | 2734.8ms/step | 21.9 samples/s
|
| 498 |
+
Step 19330/50462 | Loss: 2.6558 | LR: 3.47e-04 | 2733.3ms/step | 22.0 samples/s
|
| 499 |
+
Step 19340/50462 | Loss: 2.6690 | LR: 3.46e-04 | 2745.0ms/step | 21.9 samples/s
|
| 500 |
+
Step 19350/50462 | Loss: 2.6327 | LR: 3.46e-04 | 2739.8ms/step | 21.9 samples/s
|
| 501 |
+
Step 19360/50462 | Loss: 2.6336 | LR: 3.46e-04 | 2737.0ms/step | 21.9 samples/s
|
| 502 |
+
Step 19370/50462 | Loss: 2.7082 | LR: 3.46e-04 | 2740.3ms/step | 21.9 samples/s
|
| 503 |
+
Step 19380/50462 | Loss: 2.6353 | LR: 3.46e-04 | 2768.5ms/step | 21.7 samples/s
|
| 504 |
+
Step 19390/50462 | Loss: 2.6185 | LR: 3.46e-04 | 2757.6ms/step | 21.8 samples/s
|
| 505 |
+
Step 19400/50462 | Loss: 2.6284 | LR: 3.46e-04 | 2755.9ms/step | 21.8 samples/s
|
| 506 |
+
Step 19410/50462 | Loss: 2.6367 | LR: 3.45e-04 | 2747.6ms/step | 21.8 samples/s
|
| 507 |
+
Step 19420/50462 | Loss: 2.6211 | LR: 3.45e-04 | 2759.2ms/step | 21.7 samples/s
|
| 508 |
+
Step 19430/50462 | Loss: 2.6264 | LR: 3.45e-04 | 2751.1ms/step | 21.8 samples/s
|
| 509 |
+
Step 19440/50462 | Loss: 2.6321 | LR: 3.45e-04 | 2747.0ms/step | 21.8 samples/s
|
| 510 |
+
Step 19450/50462 | Loss: 2.6392 | LR: 3.45e-04 | 2747.6ms/step | 21.8 samples/s
|
| 511 |
+
Step 19460/50462 | Loss: 2.6357 | LR: 3.45e-04 | 2753.7ms/step | 21.8 samples/s
|
| 512 |
+
Step 19470/50462 | Loss: 2.6179 | LR: 3.45e-04 | 2758.6ms/step | 21.8 samples/s
|
| 513 |
+
Step 19480/50462 | Loss: 2.6813 | LR: 3.44e-04 | 2744.7ms/step | 21.9 samples/s
|
| 514 |
+
Step 19490/50462 | Loss: 2.6897 | LR: 3.44e-04 | 2745.4ms/step | 21.9 samples/s
|
| 515 |
+
Step 19500/50462 | Loss: 2.6153 | LR: 3.44e-04 | 2750.4ms/step | 21.8 samples/s
|
| 516 |
+
Validation at step 19500: Loss = 2.6464, Perplexity = 14.10
|
| 517 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19500
|
| 518 |
+
Step 19510/50462 | Loss: 2.6549 | LR: 3.44e-04 | 4187.4ms/step | 14.3 samples/s
|
| 519 |
+
Step 19520/50462 | Loss: 2.5752 | LR: 3.44e-04 | 2740.7ms/step | 21.9 samples/s
|
| 520 |
+
Step 19530/50462 | Loss: 2.6645 | LR: 3.44e-04 | 2757.0ms/step | 21.8 samples/s
|
| 521 |
+
Step 19540/50462 | Loss: 2.6547 | LR: 3.44e-04 | 2750.1ms/step | 21.8 samples/s
|
| 522 |
+
Step 19550/50462 | Loss: 2.6796 | LR: 3.43e-04 | 2741.8ms/step | 21.9 samples/s
|
| 523 |
+
Step 19560/50462 | Loss: 2.6689 | LR: 3.43e-04 | 2744.6ms/step | 21.9 samples/s
|
| 524 |
+
Step 19570/50462 | Loss: 2.6225 | LR: 3.43e-04 | 2740.4ms/step | 21.9 samples/s
|
| 525 |
+
Step 19580/50462 | Loss: 2.6871 | LR: 3.43e-04 | 2736.8ms/step | 21.9 samples/s
|
| 526 |
+
Step 19590/50462 | Loss: 2.6407 | LR: 3.43e-04 | 2736.0ms/step | 21.9 samples/s
|
| 527 |
+
Step 19600/50462 | Loss: 2.6722 | LR: 3.43e-04 | 2743.9ms/step | 21.9 samples/s
|
| 528 |
+
Step 19610/50462 | Loss: 2.6735 | LR: 3.42e-04 | 2737.5ms/step | 21.9 samples/s
|
| 529 |
+
Step 19620/50462 | Loss: 2.6257 | LR: 3.42e-04 | 2740.9ms/step | 21.9 samples/s
|
| 530 |
+
Step 19630/50462 | Loss: 2.6467 | LR: 3.42e-04 | 2740.4ms/step | 21.9 samples/s
|
| 531 |
+
Step 19640/50462 | Loss: 2.6254 | LR: 3.42e-04 | 2749.0ms/step | 21.8 samples/s
|
| 532 |
+
Step 19650/50462 | Loss: 2.6664 | LR: 3.42e-04 | 2745.2ms/step | 21.9 samples/s
|
| 533 |
+
Step 19660/50462 | Loss: 2.6260 | LR: 3.42e-04 | 2744.2ms/step | 21.9 samples/s
|
| 534 |
+
Step 19670/50462 | Loss: 2.6469 | LR: 3.42e-04 | 2729.6ms/step | 22.0 samples/s
|
| 535 |
+
Step 19680/50462 | Loss: 2.6301 | LR: 3.41e-04 | 2741.3ms/step | 21.9 samples/s
|
| 536 |
+
Step 19690/50462 | Loss: 2.6725 | LR: 3.41e-04 | 2744.2ms/step | 21.9 samples/s
|
| 537 |
+
Step 19700/50462 | Loss: 2.6481 | LR: 3.41e-04 | 2750.0ms/step | 21.8 samples/s
|
| 538 |
+
Step 19710/50462 | Loss: 2.6438 | LR: 3.41e-04 | 2753.0ms/step | 21.8 samples/s
|
| 539 |
+
Step 19720/50462 | Loss: 2.6878 | LR: 3.41e-04 | 2743.8ms/step | 21.9 samples/s
|
| 540 |
+
Step 19730/50462 | Loss: 2.6475 | LR: 3.41e-04 | 2734.0ms/step | 21.9 samples/s
|
| 541 |
+
Step 19740/50462 | Loss: 2.6744 | LR: 3.41e-04 | 2743.0ms/step | 21.9 samples/s
|
| 542 |
+
Step 19750/50462 | Loss: 2.6847 | LR: 3.40e-04 | 2740.9ms/step | 21.9 samples/s
|
| 543 |
+
Validation at step 19750: Loss = 2.6442, Perplexity = 14.07
|
| 544 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-19750
|
| 545 |
+
Step 19760/50462 | Loss: 2.6104 | LR: 3.40e-04 | 4156.0ms/step | 14.4 samples/s
|
| 546 |
+
Step 19770/50462 | Loss: 2.6588 | LR: 3.40e-04 | 2748.4ms/step | 21.8 samples/s
|
| 547 |
+
Step 19780/50462 | Loss: 2.6679 | LR: 3.40e-04 | 2751.8ms/step | 21.8 samples/s
|
| 548 |
+
Step 19790/50462 | Loss: 2.6127 | LR: 3.40e-04 | 2738.3ms/step | 21.9 samples/s
|
| 549 |
+
Step 19800/50462 | Loss: 2.6541 | LR: 3.40e-04 | 2738.5ms/step | 21.9 samples/s
|
| 550 |
+
Step 19810/50462 | Loss: 2.6461 | LR: 3.40e-04 | 2746.5ms/step | 21.8 samples/s
|
| 551 |
+
Step 19820/50462 | Loss: 2.6442 | LR: 3.39e-04 | 2741.2ms/step | 21.9 samples/s
|
| 552 |
+
Step 19830/50462 | Loss: 2.6599 | LR: 3.39e-04 | 2740.8ms/step | 21.9 samples/s
|
| 553 |
+
Step 19840/50462 | Loss: 2.6507 | LR: 3.39e-04 | 2744.8ms/step | 21.9 samples/s
|
| 554 |
+
Step 19850/50462 | Loss: 2.6332 | LR: 3.39e-04 | 2748.7ms/step | 21.8 samples/s
|
| 555 |
+
Step 19860/50462 | Loss: 2.6273 | LR: 3.39e-04 | 2744.8ms/step | 21.9 samples/s
|
| 556 |
+
Step 19870/50462 | Loss: 2.6322 | LR: 3.39e-04 | 2738.2ms/step | 21.9 samples/s
|
| 557 |
+
Step 19880/50462 | Loss: 2.6419 | LR: 3.39e-04 | 2746.5ms/step | 21.8 samples/s
|
| 558 |
+
Step 19890/50462 | Loss: 2.6561 | LR: 3.38e-04 | 2745.8ms/step | 21.9 samples/s
|
| 559 |
+
Step 19900/50462 | Loss: 2.6247 | LR: 3.38e-04 | 2737.8ms/step | 21.9 samples/s
|
| 560 |
+
Step 19910/50462 | Loss: 2.6649 | LR: 3.38e-04 | 2739.0ms/step | 21.9 samples/s
|
| 561 |
+
Step 19920/50462 | Loss: 2.6307 | LR: 3.38e-04 | 2744.8ms/step | 21.9 samples/s
|
| 562 |
+
Step 19930/50462 | Loss: 2.6360 | LR: 3.38e-04 | 2752.0ms/step | 21.8 samples/s
|
| 563 |
+
Step 19940/50462 | Loss: 2.6153 | LR: 3.38e-04 | 2755.8ms/step | 21.8 samples/s
|
| 564 |
+
Step 19950/50462 | Loss: 2.6410 | LR: 3.37e-04 | 2743.0ms/step | 21.9 samples/s
|
| 565 |
+
Step 19960/50462 | Loss: 2.6531 | LR: 3.37e-04 | 2737.4ms/step | 21.9 samples/s
|
| 566 |
+
Step 19970/50462 | Loss: 2.6188 | LR: 3.37e-04 | 2748.7ms/step | 21.8 samples/s
|
| 567 |
+
Step 19980/50462 | Loss: 2.6537 | LR: 3.37e-04 | 2748.8ms/step | 21.8 samples/s
|
| 568 |
+
Step 19990/50462 | Loss: 2.6197 | LR: 3.37e-04 | 2742.3ms/step | 21.9 samples/s
|
| 569 |
+
Step 20000/50462 | Loss: 2.6187 | LR: 3.37e-04 | 2735.7ms/step | 21.9 samples/s
|
| 570 |
+
Validation at step 20000: Loss = 2.6416, Perplexity = 14.04
|
| 571 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20000
|
| 572 |
+
Step 20010/50462 | Loss: 2.6349 | LR: 3.37e-04 | 4124.5ms/step | 14.5 samples/s
|
| 573 |
+
Step 20020/50462 | Loss: 2.6578 | LR: 3.36e-04 | 2741.0ms/step | 21.9 samples/s
|
| 574 |
+
Step 20030/50462 | Loss: 2.6743 | LR: 3.36e-04 | 2747.1ms/step | 21.8 samples/s
|
| 575 |
+
Step 20040/50462 | Loss: 2.6234 | LR: 3.36e-04 | 2745.5ms/step | 21.9 samples/s
|
| 576 |
+
Step 20050/50462 | Loss: 2.6694 | LR: 3.36e-04 | 2738.8ms/step | 21.9 samples/s
|
| 577 |
+
Step 20060/50462 | Loss: 2.6619 | LR: 3.36e-04 | 2744.1ms/step | 21.9 samples/s
|
| 578 |
+
Step 20070/50462 | Loss: 2.6483 | LR: 3.36e-04 | 2747.5ms/step | 21.8 samples/s
|
| 579 |
+
Step 20080/50462 | Loss: 2.6292 | LR: 3.36e-04 | 2745.9ms/step | 21.9 samples/s
|
| 580 |
+
Step 20090/50462 | Loss: 2.6646 | LR: 3.35e-04 | 2745.6ms/step | 21.9 samples/s
|
| 581 |
+
Step 20100/50462 | Loss: 2.6342 | LR: 3.35e-04 | 2744.1ms/step | 21.9 samples/s
|
| 582 |
+
Step 20110/50462 | Loss: 2.6467 | LR: 3.35e-04 | 2735.0ms/step | 21.9 samples/s
|
| 583 |
+
Step 20120/50462 | Loss: 2.6245 | LR: 3.35e-04 | 2748.6ms/step | 21.8 samples/s
|
| 584 |
+
Step 20130/50462 | Loss: 2.6186 | LR: 3.35e-04 | 2743.1ms/step | 21.9 samples/s
|
| 585 |
+
Step 20140/50462 | Loss: 2.6173 | LR: 3.35e-04 | 2749.7ms/step | 21.8 samples/s
|
| 586 |
+
Step 20150/50462 | Loss: 2.6647 | LR: 3.35e-04 | 2752.9ms/step | 21.8 samples/s
|
| 587 |
+
Step 20160/50462 | Loss: 2.5865 | LR: 3.34e-04 | 2742.4ms/step | 21.9 samples/s
|
| 588 |
+
Step 20170/50462 | Loss: 2.6214 | LR: 3.34e-04 | 2744.4ms/step | 21.9 samples/s
|
| 589 |
+
Step 20180/50462 | Loss: 2.5806 | LR: 3.34e-04 | 2749.6ms/step | 21.8 samples/s
|
| 590 |
+
Step 20190/50462 | Loss: 2.6364 | LR: 3.34e-04 | 2743.0ms/step | 21.9 samples/s
|
| 591 |
+
Step 20200/50462 | Loss: 2.6392 | LR: 3.34e-04 | 2746.1ms/step | 21.8 samples/s
|
| 592 |
+
Step 20210/50462 | Loss: 2.6154 | LR: 3.34e-04 | 2756.0ms/step | 21.8 samples/s
|
| 593 |
+
Step 20220/50462 | Loss: 2.6140 | LR: 3.33e-04 | 2739.5ms/step | 21.9 samples/s
|
| 594 |
+
Step 20230/50462 | Loss: 2.6760 | LR: 3.33e-04 | 2743.0ms/step | 21.9 samples/s
|
| 595 |
+
Step 20240/50462 | Loss: 2.6766 | LR: 3.33e-04 | 2734.0ms/step | 21.9 samples/s
|
| 596 |
+
Step 20250/50462 | Loss: 2.6327 | LR: 3.33e-04 | 2746.3ms/step | 21.8 samples/s
|
| 597 |
+
Validation at step 20250: Loss = 2.6395, Perplexity = 14.01
|
| 598 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20250
|
| 599 |
+
Step 20260/50462 | Loss: 2.6432 | LR: 3.33e-04 | 4112.8ms/step | 14.6 samples/s
|
| 600 |
+
Step 20270/50462 | Loss: 2.6299 | LR: 3.33e-04 | 2748.3ms/step | 21.8 samples/s
|
| 601 |
+
Step 20280/50462 | Loss: 2.5800 | LR: 3.33e-04 | 2746.0ms/step | 21.9 samples/s
|
| 602 |
+
Step 20290/50462 | Loss: 2.6297 | LR: 3.32e-04 | 2745.5ms/step | 21.9 samples/s
|
| 603 |
+
Step 20300/50462 | Loss: 2.6427 | LR: 3.32e-04 | 2740.1ms/step | 21.9 samples/s
|
| 604 |
+
Step 20310/50462 | Loss: 2.6623 | LR: 3.32e-04 | 2744.4ms/step | 21.9 samples/s
|
| 605 |
+
Step 20320/50462 | Loss: 2.6565 | LR: 3.32e-04 | 2750.3ms/step | 21.8 samples/s
|
| 606 |
+
Step 20330/50462 | Loss: 2.6644 | LR: 3.32e-04 | 2749.8ms/step | 21.8 samples/s
|
| 607 |
+
Step 20340/50462 | Loss: 2.6502 | LR: 3.32e-04 | 2748.5ms/step | 21.8 samples/s
|
| 608 |
+
Step 20350/50462 | Loss: 2.6573 | LR: 3.32e-04 | 2749.2ms/step | 21.8 samples/s
|
| 609 |
+
Step 20360/50462 | Loss: 2.6312 | LR: 3.31e-04 | 2743.9ms/step | 21.9 samples/s
|
| 610 |
+
Step 20370/50462 | Loss: 2.6345 | LR: 3.31e-04 | 2747.3ms/step | 21.8 samples/s
|
| 611 |
+
Step 20380/50462 | Loss: 2.6714 | LR: 3.31e-04 | 2755.9ms/step | 21.8 samples/s
|
| 612 |
+
Step 20390/50462 | Loss: 2.6111 | LR: 3.31e-04 | 2746.9ms/step | 21.8 samples/s
|
| 613 |
+
Step 20400/50462 | Loss: 2.6295 | LR: 3.31e-04 | 2745.4ms/step | 21.9 samples/s
|
| 614 |
+
Step 20410/50462 | Loss: 2.6130 | LR: 3.31e-04 | 2750.8ms/step | 21.8 samples/s
|
| 615 |
+
Step 20420/50462 | Loss: 2.6510 | LR: 3.30e-04 | 2745.5ms/step | 21.9 samples/s
|
| 616 |
+
Step 20430/50462 | Loss: 2.6519 | LR: 3.30e-04 | 2739.8ms/step | 21.9 samples/s
|
| 617 |
+
Step 20440/50462 | Loss: 2.5872 | LR: 3.30e-04 | 2750.8ms/step | 21.8 samples/s
|
| 618 |
+
Step 20450/50462 | Loss: 2.6059 | LR: 3.30e-04 | 2745.1ms/step | 21.9 samples/s
|
| 619 |
+
Step 20460/50462 | Loss: 2.6290 | LR: 3.30e-04 | 2737.4ms/step | 21.9 samples/s
|
| 620 |
+
Step 20470/50462 | Loss: 2.6615 | LR: 3.30e-04 | 2747.6ms/step | 21.8 samples/s
|
| 621 |
+
Step 20480/50462 | Loss: 2.6529 | LR: 3.30e-04 | 2751.9ms/step | 21.8 samples/s
|
| 622 |
+
Step 20490/50462 | Loss: 2.6639 | LR: 3.29e-04 | 2751.0ms/step | 21.8 samples/s
|
| 623 |
+
Step 20500/50462 | Loss: 2.6275 | LR: 3.29e-04 | 2752.1ms/step | 21.8 samples/s
|
| 624 |
+
Validation at step 20500: Loss = 2.6379, Perplexity = 13.98
|
| 625 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20500
|
| 626 |
+
Step 20510/50462 | Loss: 2.6431 | LR: 3.29e-04 | 4139.4ms/step | 14.5 samples/s
|
| 627 |
+
Step 20520/50462 | Loss: 2.6414 | LR: 3.29e-04 | 2752.5ms/step | 21.8 samples/s
|
| 628 |
+
Step 20530/50462 | Loss: 2.6104 | LR: 3.29e-04 | 2748.6ms/step | 21.8 samples/s
|
| 629 |
+
Step 20540/50462 | Loss: 2.6352 | LR: 3.29e-04 | 2750.5ms/step | 21.8 samples/s
|
| 630 |
+
Step 20550/50462 | Loss: 2.6229 | LR: 3.29e-04 | 2747.2ms/step | 21.8 samples/s
|
| 631 |
+
Step 20560/50462 | Loss: 2.6329 | LR: 3.28e-04 | 2743.5ms/step | 21.9 samples/s
|
| 632 |
+
Step 20570/50462 | Loss: 2.6224 | LR: 3.28e-04 | 2747.9ms/step | 21.8 samples/s
|
| 633 |
+
Step 20580/50462 | Loss: 2.6400 | LR: 3.28e-04 | 2743.1ms/step | 21.9 samples/s
|
| 634 |
+
Step 20590/50462 | Loss: 2.6448 | LR: 3.28e-04 | 2746.8ms/step | 21.8 samples/s
|
| 635 |
+
Step 20600/50462 | Loss: 2.6504 | LR: 3.28e-04 | 2743.1ms/step | 21.9 samples/s
|
| 636 |
+
Step 20610/50462 | Loss: 2.6678 | LR: 3.28e-04 | 2753.0ms/step | 21.8 samples/s
|
| 637 |
+
Step 20620/50462 | Loss: 2.6032 | LR: 3.27e-04 | 2749.1ms/step | 21.8 samples/s
|
| 638 |
+
Step 20630/50462 | Loss: 2.6442 | LR: 3.27e-04 | 2745.8ms/step | 21.9 samples/s
|
| 639 |
+
Step 20640/50462 | Loss: 2.6657 | LR: 3.27e-04 | 2739.0ms/step | 21.9 samples/s
|
| 640 |
+
Step 20650/50462 | Loss: 2.6376 | LR: 3.27e-04 | 2746.1ms/step | 21.8 samples/s
|
| 641 |
+
Step 20660/50462 | Loss: 2.6566 | LR: 3.27e-04 | 2743.4ms/step | 21.9 samples/s
|
| 642 |
+
Step 20670/50462 | Loss: 2.6494 | LR: 3.27e-04 | 2756.7ms/step | 21.8 samples/s
|
| 643 |
+
Step 20680/50462 | Loss: 2.6050 | LR: 3.27e-04 | 2744.9ms/step | 21.9 samples/s
|
| 644 |
+
Step 20690/50462 | Loss: 2.6466 | LR: 3.26e-04 | 2746.5ms/step | 21.8 samples/s
|
| 645 |
+
Step 20700/50462 | Loss: 2.6591 | LR: 3.26e-04 | 2745.4ms/step | 21.9 samples/s
|
| 646 |
+
Step 20710/50462 | Loss: 2.6571 | LR: 3.26e-04 | 2742.9ms/step | 21.9 samples/s
|
| 647 |
+
Step 20720/50462 | Loss: 2.6123 | LR: 3.26e-04 | 2742.4ms/step | 21.9 samples/s
|
| 648 |
+
Step 20730/50462 | Loss: 2.6722 | LR: 3.26e-04 | 2739.9ms/step | 21.9 samples/s
|
| 649 |
+
Step 20740/50462 | Loss: 2.6118 | LR: 3.26e-04 | 2741.5ms/step | 21.9 samples/s
|
| 650 |
+
Step 20750/50462 | Loss: 2.6206 | LR: 3.26e-04 | 2759.7ms/step | 21.7 samples/s
|
| 651 |
+
Validation at step 20750: Loss = 2.6356, Perplexity = 13.95
|
| 652 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-20750
|
| 653 |
+
Step 20760/50462 | Loss: 2.6315 | LR: 3.25e-04 | 4129.2ms/step | 14.5 samples/s
|
| 654 |
+
Step 20770/50462 | Loss: 2.6317 | LR: 3.25e-04 | 2741.4ms/step | 21.9 samples/s
|
| 655 |
+
Step 20780/50462 | Loss: 2.6625 | LR: 3.25e-04 | 2736.2ms/step | 21.9 samples/s
|
| 656 |
+
Step 20790/50462 | Loss: 2.6452 | LR: 3.25e-04 | 2745.4ms/step | 21.9 samples/s
|
| 657 |
+
Step 20800/50462 | Loss: 2.6462 | LR: 3.25e-04 | 2740.5ms/step | 21.9 samples/s
|
| 658 |
+
Step 20810/50462 | Loss: 2.5724 | LR: 3.25e-04 | 2747.5ms/step | 21.8 samples/s
|
| 659 |
+
Step 20820/50462 | Loss: 2.6278 | LR: 3.24e-04 | 2741.8ms/step | 21.9 samples/s
|
| 660 |
+
Step 20830/50462 | Loss: 2.6719 | LR: 3.24e-04 | 2741.1ms/step | 21.9 samples/s
|
| 661 |
+
Step 20840/50462 | Loss: 2.6066 | LR: 3.24e-04 | 2747.6ms/step | 21.8 samples/s
|
| 662 |
+
Step 20850/50462 | Loss: 2.6700 | LR: 3.24e-04 | 2743.0ms/step | 21.9 samples/s
|
| 663 |
+
Step 20860/50462 | Loss: 2.6401 | LR: 3.24e-04 | 2748.8ms/step | 21.8 samples/s
|
| 664 |
+
Step 20870/50462 | Loss: 2.6319 | LR: 3.24e-04 | 2751.7ms/step | 21.8 samples/s
|
| 665 |
+
Step 20880/50462 | Loss: 2.6221 | LR: 3.24e-04 | 2737.9ms/step | 21.9 samples/s
|
| 666 |
+
Step 20890/50462 | Loss: 2.6450 | LR: 3.23e-04 | 2744.3ms/step | 21.9 samples/s
|
| 667 |
+
Step 20900/50462 | Loss: 2.6214 | LR: 3.23e-04 | 2746.5ms/step | 21.8 samples/s
|
| 668 |
+
Step 20910/50462 | Loss: 2.6581 | LR: 3.23e-04 | 2747.4ms/step | 21.8 samples/s
|
| 669 |
+
Step 20920/50462 | Loss: 2.6554 | LR: 3.23e-04 | 2748.7ms/step | 21.8 samples/s
|
| 670 |
+
Step 20930/50462 | Loss: 2.6590 | LR: 3.23e-04 | 2746.5ms/step | 21.8 samples/s
|
| 671 |
+
Step 20940/50462 | Loss: 2.5726 | LR: 3.23e-04 | 2745.9ms/step | 21.9 samples/s
|
| 672 |
+
Step 20950/50462 | Loss: 2.6213 | LR: 3.23e-04 | 2745.3ms/step | 21.9 samples/s
|
| 673 |
+
Step 20960/50462 | Loss: 2.6527 | LR: 3.22e-04 | 2737.3ms/step | 21.9 samples/s
|
| 674 |
+
Step 20970/50462 | Loss: 2.6566 | LR: 3.22e-04 | 2747.2ms/step | 21.8 samples/s
|
| 675 |
+
Step 20980/50462 | Loss: 2.6400 | LR: 3.22e-04 | 2753.4ms/step | 21.8 samples/s
|
| 676 |
+
Step 20990/50462 | Loss: 2.6241 | LR: 3.22e-04 | 2746.4ms/step | 21.8 samples/s
|
| 677 |
+
Step 21000/50462 | Loss: 2.6749 | LR: 3.22e-04 | 2741.1ms/step | 21.9 samples/s
|
| 678 |
+
Validation at step 21000: Loss = 2.6333, Perplexity = 13.92
|
| 679 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21000
|
| 680 |
+
Step 21010/50462 | Loss: 2.6459 | LR: 3.22e-04 | 4151.9ms/step | 14.5 samples/s
|
| 681 |
+
Step 21020/50462 | Loss: 2.6339 | LR: 3.21e-04 | 2739.5ms/step | 21.9 samples/s
|
| 682 |
+
Step 21030/50462 | Loss: 2.6527 | LR: 3.21e-04 | 2753.2ms/step | 21.8 samples/s
|
| 683 |
+
Step 21040/50462 | Loss: 2.6239 | LR: 3.21e-04 | 2744.8ms/step | 21.9 samples/s
|
| 684 |
+
Step 21050/50462 | Loss: 2.6492 | LR: 3.21e-04 | 2746.8ms/step | 21.8 samples/s
|
| 685 |
+
Step 21060/50462 | Loss: 2.6417 | LR: 3.21e-04 | 2747.8ms/step | 21.8 samples/s
|
| 686 |
+
Step 21070/50462 | Loss: 2.6265 | LR: 3.21e-04 | 2742.7ms/step | 21.9 samples/s
|
| 687 |
+
Step 21080/50462 | Loss: 2.6315 | LR: 3.21e-04 | 2741.2ms/step | 21.9 samples/s
|
| 688 |
+
Step 21090/50462 | Loss: 2.6467 | LR: 3.20e-04 | 2742.3ms/step | 21.9 samples/s
|
| 689 |
+
Step 21100/50462 | Loss: 2.6455 | LR: 3.20e-04 | 2754.3ms/step | 21.8 samples/s
|
| 690 |
+
Step 21110/50462 | Loss: 2.5656 | LR: 3.20e-04 | 2750.3ms/step | 21.8 samples/s
|
| 691 |
+
Step 21120/50462 | Loss: 2.6083 | LR: 3.20e-04 | 2738.9ms/step | 21.9 samples/s
|
| 692 |
+
Step 21130/50462 | Loss: 2.6556 | LR: 3.20e-04 | 2745.0ms/step | 21.9 samples/s
|
| 693 |
+
Step 21140/50462 | Loss: 2.6538 | LR: 3.20e-04 | 2741.8ms/step | 21.9 samples/s
|
| 694 |
+
Step 21150/50462 | Loss: 2.6317 | LR: 3.19e-04 | 2735.7ms/step | 21.9 samples/s
|
| 695 |
+
Step 21160/50462 | Loss: 2.5922 | LR: 3.19e-04 | 2750.2ms/step | 21.8 samples/s
|
| 696 |
+
Step 21170/50462 | Loss: 2.6330 | LR: 3.19e-04 | 2740.3ms/step | 21.9 samples/s
|
| 697 |
+
Step 21180/50462 | Loss: 2.6665 | LR: 3.19e-04 | 2741.7ms/step | 21.9 samples/s
|
| 698 |
+
Step 21190/50462 | Loss: 2.6266 | LR: 3.19e-04 | 2745.6ms/step | 21.9 samples/s
|
| 699 |
+
Step 21200/50462 | Loss: 2.6091 | LR: 3.19e-04 | 2740.1ms/step | 21.9 samples/s
|
| 700 |
+
Step 21210/50462 | Loss: 2.6392 | LR: 3.19e-04 | 2742.6ms/step | 21.9 samples/s
|
| 701 |
+
Step 21220/50462 | Loss: 2.6085 | LR: 3.18e-04 | 2745.5ms/step | 21.9 samples/s
|
| 702 |
+
Step 21230/50462 | Loss: 2.6369 | LR: 3.18e-04 | 2735.1ms/step | 21.9 samples/s
|
| 703 |
+
Step 21240/50462 | Loss: 2.6343 | LR: 3.18e-04 | 2739.6ms/step | 21.9 samples/s
|
| 704 |
+
Step 21250/50462 | Loss: 2.6423 | LR: 3.18e-04 | 2740.1ms/step | 21.9 samples/s
|
| 705 |
+
Validation at step 21250: Loss = 2.6301, Perplexity = 13.87
|
| 706 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21250
|
| 707 |
+
Step 21260/50462 | Loss: 2.6793 | LR: 3.18e-04 | 4130.8ms/step | 14.5 samples/s
|
| 708 |
+
Step 21270/50462 | Loss: 2.6500 | LR: 3.18e-04 | 2739.7ms/step | 21.9 samples/s
|
| 709 |
+
Step 21280/50462 | Loss: 2.6139 | LR: 3.17e-04 | 2746.8ms/step | 21.8 samples/s
|
| 710 |
+
Step 21290/50462 | Loss: 2.6811 | LR: 3.17e-04 | 2747.2ms/step | 21.8 samples/s
|
| 711 |
+
Step 21300/50462 | Loss: 2.6074 | LR: 3.17e-04 | 2734.3ms/step | 21.9 samples/s
|
| 712 |
+
Step 21310/50462 | Loss: 2.6130 | LR: 3.17e-04 | 2748.2ms/step | 21.8 samples/s
|
| 713 |
+
Step 21320/50462 | Loss: 2.6454 | LR: 3.17e-04 | 2735.1ms/step | 21.9 samples/s
|
| 714 |
+
Step 21330/50462 | Loss: 2.6593 | LR: 3.17e-04 | 2749.4ms/step | 21.8 samples/s
|
| 715 |
+
Step 21340/50462 | Loss: 2.6823 | LR: 3.17e-04 | 2747.6ms/step | 21.8 samples/s
|
| 716 |
+
Step 21350/50462 | Loss: 2.6067 | LR: 3.16e-04 | 2750.6ms/step | 21.8 samples/s
|
| 717 |
+
Step 21360/50462 | Loss: 2.5933 | LR: 3.16e-04 | 2735.8ms/step | 21.9 samples/s
|
| 718 |
+
Step 21370/50462 | Loss: 2.6192 | LR: 3.16e-04 | 2744.1ms/step | 21.9 samples/s
|
| 719 |
+
Step 21380/50462 | Loss: 2.6514 | LR: 3.16e-04 | 2742.7ms/step | 21.9 samples/s
|
| 720 |
+
Step 21390/50462 | Loss: 2.6398 | LR: 3.16e-04 | 2744.1ms/step | 21.9 samples/s
|
| 721 |
+
Step 21400/50462 | Loss: 2.6140 | LR: 3.16e-04 | 2742.3ms/step | 21.9 samples/s
|
| 722 |
+
Step 21410/50462 | Loss: 2.6531 | LR: 3.16e-04 | 2741.5ms/step | 21.9 samples/s
|
| 723 |
+
Step 21420/50462 | Loss: 2.5842 | LR: 3.15e-04 | 2750.8ms/step | 21.8 samples/s
|
| 724 |
+
Step 21430/50462 | Loss: 2.5973 | LR: 3.15e-04 | 2740.2ms/step | 21.9 samples/s
|
| 725 |
+
Step 21440/50462 | Loss: 2.6398 | LR: 3.15e-04 | 2737.9ms/step | 21.9 samples/s
|
| 726 |
+
Step 21450/50462 | Loss: 2.5989 | LR: 3.15e-04 | 2747.9ms/step | 21.8 samples/s
|
| 727 |
+
Step 21460/50462 | Loss: 2.6326 | LR: 3.15e-04 | 2747.4ms/step | 21.8 samples/s
|
| 728 |
+
Step 21470/50462 | Loss: 2.6256 | LR: 3.15e-04 | 2754.6ms/step | 21.8 samples/s
|
| 729 |
+
Step 21480/50462 | Loss: 2.6065 | LR: 3.14e-04 | 2745.8ms/step | 21.9 samples/s
|
| 730 |
+
Step 21490/50462 | Loss: 2.6349 | LR: 3.14e-04 | 2745.8ms/step | 21.9 samples/s
|
| 731 |
+
Step 21500/50462 | Loss: 2.6193 | LR: 3.14e-04 | 2742.2ms/step | 21.9 samples/s
|
| 732 |
+
Validation at step 21500: Loss = 2.6269, Perplexity = 13.83
|
| 733 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21500
|
| 734 |
+
Step 21510/50462 | Loss: 2.6076 | LR: 3.14e-04 | 4137.9ms/step | 14.5 samples/s
|
| 735 |
+
Step 21520/50462 | Loss: 2.6034 | LR: 3.14e-04 | 2732.0ms/step | 22.0 samples/s
|
| 736 |
+
Step 21530/50462 | Loss: 2.6098 | LR: 3.14e-04 | 2732.0ms/step | 22.0 samples/s
|
| 737 |
+
Step 21540/50462 | Loss: 2.6340 | LR: 3.14e-04 | 2752.3ms/step | 21.8 samples/s
|
| 738 |
+
Step 21550/50462 | Loss: 2.5621 | LR: 3.13e-04 | 2740.7ms/step | 21.9 samples/s
|
| 739 |
+
Step 21560/50462 | Loss: 2.6294 | LR: 3.13e-04 | 2738.9ms/step | 21.9 samples/s
|
| 740 |
+
Step 21570/50462 | Loss: 2.6834 | LR: 3.13e-04 | 2749.0ms/step | 21.8 samples/s
|
| 741 |
+
Step 21580/50462 | Loss: 2.6112 | LR: 3.13e-04 | 2739.6ms/step | 21.9 samples/s
|
| 742 |
+
Step 21590/50462 | Loss: 2.6126 | LR: 3.13e-04 | 2743.7ms/step | 21.9 samples/s
|
| 743 |
+
Step 21600/50462 | Loss: 2.6056 | LR: 3.13e-04 | 2752.2ms/step | 21.8 samples/s
|
| 744 |
+
Step 21610/50462 | Loss: 2.6146 | LR: 3.12e-04 | 2750.3ms/step | 21.8 samples/s
|
| 745 |
+
Step 21620/50462 | Loss: 2.6243 | LR: 3.12e-04 | 2745.7ms/step | 21.9 samples/s
|
| 746 |
+
Step 21630/50462 | Loss: 2.6064 | LR: 3.12e-04 | 2737.9ms/step | 21.9 samples/s
|
| 747 |
+
Step 21640/50462 | Loss: 2.6573 | LR: 3.12e-04 | 2746.1ms/step | 21.8 samples/s
|
| 748 |
+
Step 21650/50462 | Loss: 2.6310 | LR: 3.12e-04 | 2747.0ms/step | 21.8 samples/s
|
| 749 |
+
Step 21660/50462 | Loss: 2.6704 | LR: 3.12e-04 | 2741.1ms/step | 21.9 samples/s
|
| 750 |
+
Step 21670/50462 | Loss: 2.5794 | LR: 3.12e-04 | 2739.7ms/step | 21.9 samples/s
|
| 751 |
+
Step 21680/50462 | Loss: 2.7129 | LR: 3.11e-04 | 2737.4ms/step | 21.9 samples/s
|
| 752 |
+
Step 21690/50462 | Loss: 2.6355 | LR: 3.11e-04 | 2743.4ms/step | 21.9 samples/s
|
| 753 |
+
Step 21700/50462 | Loss: 2.6279 | LR: 3.11e-04 | 2741.5ms/step | 21.9 samples/s
|
| 754 |
+
Step 21710/50462 | Loss: 2.6621 | LR: 3.11e-04 | 2745.5ms/step | 21.9 samples/s
|
| 755 |
+
Step 21720/50462 | Loss: 2.6081 | LR: 3.11e-04 | 2746.8ms/step | 21.8 samples/s
|
| 756 |
+
Step 21730/50462 | Loss: 2.6302 | LR: 3.11e-04 | 2740.8ms/step | 21.9 samples/s
|
| 757 |
+
Step 21740/50462 | Loss: 2.5968 | LR: 3.10e-04 | 2748.9ms/step | 21.8 samples/s
|
| 758 |
+
Step 21750/50462 | Loss: 2.6490 | LR: 3.10e-04 | 2748.5ms/step | 21.8 samples/s
|
| 759 |
+
Validation at step 21750: Loss = 2.6235, Perplexity = 13.78
|
| 760 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-21750
|
| 761 |
+
Step 21760/50462 | Loss: 2.6053 | LR: 3.10e-04 | 4136.8ms/step | 14.5 samples/s
|
| 762 |
+
Step 21770/50462 | Loss: 2.6212 | LR: 3.10e-04 | 2740.0ms/step | 21.9 samples/s
|
| 763 |
+
Step 21780/50462 | Loss: 2.6365 | LR: 3.10e-04 | 2739.4ms/step | 21.9 samples/s
|
| 764 |
+
Step 21790/50462 | Loss: 2.6033 | LR: 3.10e-04 | 2747.3ms/step | 21.8 samples/s
|
| 765 |
+
Step 21800/50462 | Loss: 2.6104 | LR: 3.10e-04 | 2741.3ms/step | 21.9 samples/s
|
| 766 |
+
Step 21810/50462 | Loss: 2.6218 | LR: 3.09e-04 | 2744.7ms/step | 21.9 samples/s
|
| 767 |
+
Step 21820/50462 | Loss: 2.5914 | LR: 3.09e-04 | 2739.3ms/step | 21.9 samples/s
|
| 768 |
+
Step 21830/50462 | Loss: 2.5989 | LR: 3.09e-04 | 2748.1ms/step | 21.8 samples/s
|
| 769 |
+
Step 21840/50462 | Loss: 2.6581 | LR: 3.09e-04 | 2746.0ms/step | 21.9 samples/s
|
| 770 |
+
Step 21850/50462 | Loss: 2.5708 | LR: 3.09e-04 | 2734.1ms/step | 21.9 samples/s
|
| 771 |
+
Step 21860/50462 | Loss: 2.6033 | LR: 3.09e-04 | 2744.0ms/step | 21.9 samples/s
|
| 772 |
+
Step 21870/50462 | Loss: 2.6187 | LR: 3.08e-04 | 2742.4ms/step | 21.9 samples/s
|
| 773 |
+
Step 21880/50462 | Loss: 2.6382 | LR: 3.08e-04 | 2740.4ms/step | 21.9 samples/s
|
| 774 |
+
Step 21890/50462 | Loss: 2.6377 | LR: 3.08e-04 | 2742.5ms/step | 21.9 samples/s
|
| 775 |
+
Step 21900/50462 | Loss: 2.6030 | LR: 3.08e-04 | 2747.2ms/step | 21.8 samples/s
|
| 776 |
+
Step 21910/50462 | Loss: 2.5579 | LR: 3.08e-04 | 2740.1ms/step | 21.9 samples/s
|
| 777 |
+
Step 21920/50462 | Loss: 2.6545 | LR: 3.08e-04 | 2746.3ms/step | 21.8 samples/s
|
| 778 |
+
Step 21930/50462 | Loss: 2.6085 | LR: 3.08e-04 | 2751.5ms/step | 21.8 samples/s
|
| 779 |
+
Step 21940/50462 | Loss: 2.6217 | LR: 3.07e-04 | 2741.7ms/step | 21.9 samples/s
|
| 780 |
+
Step 21950/50462 | Loss: 2.6008 | LR: 3.07e-04 | 2737.2ms/step | 21.9 samples/s
|
| 781 |
+
Step 21960/50462 | Loss: 2.5927 | LR: 3.07e-04 | 2739.9ms/step | 21.9 samples/s
|
| 782 |
+
Step 21970/50462 | Loss: 2.6327 | LR: 3.07e-04 | 2742.3ms/step | 21.9 samples/s
|
| 783 |
+
Step 21980/50462 | Loss: 2.6524 | LR: 3.07e-04 | 2745.1ms/step | 21.9 samples/s
|
| 784 |
+
Step 21990/50462 | Loss: 2.6330 | LR: 3.07e-04 | 2737.4ms/step | 21.9 samples/s
|
| 785 |
+
Step 22000/50462 | Loss: 2.6035 | LR: 3.06e-04 | 2738.6ms/step | 21.9 samples/s
|
| 786 |
+
Validation at step 22000: Loss = 2.6218, Perplexity = 13.76
|
| 787 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22000
|
| 788 |
+
Step 22010/50462 | Loss: 2.6054 | LR: 3.06e-04 | 4142.0ms/step | 14.5 samples/s
|
| 789 |
+
Step 22020/50462 | Loss: 2.6393 | LR: 3.06e-04 | 2744.8ms/step | 21.9 samples/s
|
| 790 |
+
Step 22030/50462 | Loss: 2.5999 | LR: 3.06e-04 | 2739.8ms/step | 21.9 samples/s
|
| 791 |
+
Step 22040/50462 | Loss: 2.6181 | LR: 3.06e-04 | 2748.9ms/step | 21.8 samples/s
|
| 792 |
+
Step 22050/50462 | Loss: 2.6324 | LR: 3.06e-04 | 2740.4ms/step | 21.9 samples/s
|
| 793 |
+
Step 22060/50462 | Loss: 2.5881 | LR: 3.06e-04 | 2744.3ms/step | 21.9 samples/s
|
| 794 |
+
Step 22070/50462 | Loss: 2.6568 | LR: 3.05e-04 | 2740.0ms/step | 21.9 samples/s
|
| 795 |
+
Step 22080/50462 | Loss: 2.6082 | LR: 3.05e-04 | 2740.1ms/step | 21.9 samples/s
|
| 796 |
+
Step 22090/50462 | Loss: 2.6628 | LR: 3.05e-04 | 2741.3ms/step | 21.9 samples/s
|
| 797 |
+
Step 22100/50462 | Loss: 2.5891 | LR: 3.05e-04 | 2744.4ms/step | 21.9 samples/s
|
| 798 |
+
Step 22110/50462 | Loss: 2.6443 | LR: 3.05e-04 | 2750.0ms/step | 21.8 samples/s
|
| 799 |
+
Step 22120/50462 | Loss: 2.6316 | LR: 3.05e-04 | 2744.3ms/step | 21.9 samples/s
|
| 800 |
+
Step 22130/50462 | Loss: 2.5886 | LR: 3.04e-04 | 2745.0ms/step | 21.9 samples/s
|
| 801 |
+
Step 22140/50462 | Loss: 2.6418 | LR: 3.04e-04 | 2746.4ms/step | 21.8 samples/s
|
| 802 |
+
Step 22150/50462 | Loss: 2.6301 | LR: 3.04e-04 | 2750.8ms/step | 21.8 samples/s
|
| 803 |
+
Step 22160/50462 | Loss: 2.6622 | LR: 3.04e-04 | 2744.5ms/step | 21.9 samples/s
|
| 804 |
+
Step 22170/50462 | Loss: 2.6287 | LR: 3.04e-04 | 2741.3ms/step | 21.9 samples/s
|
| 805 |
+
Step 22180/50462 | Loss: 2.6496 | LR: 3.04e-04 | 2737.3ms/step | 21.9 samples/s
|
| 806 |
+
Step 22190/50462 | Loss: 2.6109 | LR: 3.04e-04 | 2748.6ms/step | 21.8 samples/s
|
| 807 |
+
Step 22200/50462 | Loss: 2.5839 | LR: 3.03e-04 | 2738.7ms/step | 21.9 samples/s
|
| 808 |
+
Step 22210/50462 | Loss: 2.5803 | LR: 3.03e-04 | 2738.7ms/step | 21.9 samples/s
|
| 809 |
+
Step 22220/50462 | Loss: 2.6208 | LR: 3.03e-04 | 2746.7ms/step | 21.8 samples/s
|
| 810 |
+
Step 22230/50462 | Loss: 2.6143 | LR: 3.03e-04 | 2751.6ms/step | 21.8 samples/s
|
| 811 |
+
Step 22240/50462 | Loss: 2.6381 | LR: 3.03e-04 | 2741.2ms/step | 21.9 samples/s
|
| 812 |
+
Step 22250/50462 | Loss: 2.6007 | LR: 3.03e-04 | 2743.7ms/step | 21.9 samples/s
|
| 813 |
+
Validation at step 22250: Loss = 2.6194, Perplexity = 13.73
|
| 814 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22250
|
| 815 |
+
Step 22260/50462 | Loss: 2.6005 | LR: 3.02e-04 | 4120.1ms/step | 14.6 samples/s
|
| 816 |
+
Step 22270/50462 | Loss: 2.6306 | LR: 3.02e-04 | 2740.8ms/step | 21.9 samples/s
|
| 817 |
+
Step 22280/50462 | Loss: 2.6487 | LR: 3.02e-04 | 2745.1ms/step | 21.9 samples/s
|
| 818 |
+
Step 22290/50462 | Loss: 2.5856 | LR: 3.02e-04 | 2744.5ms/step | 21.9 samples/s
|
| 819 |
+
Step 22300/50462 | Loss: 2.6223 | LR: 3.02e-04 | 2745.7ms/step | 21.9 samples/s
|
| 820 |
+
Step 22310/50462 | Loss: 2.6192 | LR: 3.02e-04 | 2742.7ms/step | 21.9 samples/s
|
| 821 |
+
Step 22320/50462 | Loss: 2.6331 | LR: 3.02e-04 | 2750.3ms/step | 21.8 samples/s
|
| 822 |
+
Step 22330/50462 | Loss: 2.6105 | LR: 3.01e-04 | 2744.3ms/step | 21.9 samples/s
|
| 823 |
+
Step 22340/50462 | Loss: 2.6420 | LR: 3.01e-04 | 2743.0ms/step | 21.9 samples/s
|
| 824 |
+
Step 22350/50462 | Loss: 2.6249 | LR: 3.01e-04 | 2745.6ms/step | 21.9 samples/s
|
| 825 |
+
Step 22360/50462 | Loss: 2.6020 | LR: 3.01e-04 | 2742.2ms/step | 21.9 samples/s
|
| 826 |
+
Step 22370/50462 | Loss: 2.6161 | LR: 3.01e-04 | 2737.7ms/step | 21.9 samples/s
|
| 827 |
+
Step 22380/50462 | Loss: 2.6322 | LR: 3.01e-04 | 2737.1ms/step | 21.9 samples/s
|
| 828 |
+
Step 22390/50462 | Loss: 2.6000 | LR: 3.00e-04 | 2740.0ms/step | 21.9 samples/s
|
| 829 |
+
Step 22400/50462 | Loss: 2.6294 | LR: 3.00e-04 | 2738.9ms/step | 21.9 samples/s
|
| 830 |
+
Step 22410/50462 | Loss: 2.6255 | LR: 3.00e-04 | 2741.3ms/step | 21.9 samples/s
|
| 831 |
+
Step 22420/50462 | Loss: 2.6182 | LR: 3.00e-04 | 2741.0ms/step | 21.9 samples/s
|
| 832 |
+
Step 22430/50462 | Loss: 2.6425 | LR: 3.00e-04 | 2737.8ms/step | 21.9 samples/s
|
| 833 |
+
Step 22440/50462 | Loss: 2.6083 | LR: 3.00e-04 | 2734.7ms/step | 21.9 samples/s
|
| 834 |
+
Step 22450/50462 | Loss: 2.6350 | LR: 3.00e-04 | 2747.0ms/step | 21.8 samples/s
|
| 835 |
+
Step 22460/50462 | Loss: 2.6553 | LR: 2.99e-04 | 2746.0ms/step | 21.9 samples/s
|
| 836 |
+
Step 22470/50462 | Loss: 2.6262 | LR: 2.99e-04 | 2739.4ms/step | 21.9 samples/s
|
| 837 |
+
Step 22480/50462 | Loss: 2.6213 | LR: 2.99e-04 | 2737.5ms/step | 21.9 samples/s
|
| 838 |
+
Step 22490/50462 | Loss: 2.6499 | LR: 2.99e-04 | 2742.3ms/step | 21.9 samples/s
|
| 839 |
+
Step 22500/50462 | Loss: 2.6349 | LR: 2.99e-04 | 2742.0ms/step | 21.9 samples/s
|
| 840 |
+
Validation at step 22500: Loss = 2.6177, Perplexity = 13.70
|
| 841 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22500
|
| 842 |
+
Step 22510/50462 | Loss: 2.5860 | LR: 2.99e-04 | 4134.6ms/step | 14.5 samples/s
|
| 843 |
+
Step 22520/50462 | Loss: 2.6526 | LR: 2.98e-04 | 2738.8ms/step | 21.9 samples/s
|
| 844 |
+
Step 22530/50462 | Loss: 2.6159 | LR: 2.98e-04 | 2743.0ms/step | 21.9 samples/s
|
| 845 |
+
Step 22540/50462 | Loss: 2.5670 | LR: 2.98e-04 | 2739.3ms/step | 21.9 samples/s
|
| 846 |
+
Step 22550/50462 | Loss: 2.6087 | LR: 2.98e-04 | 2735.9ms/step | 21.9 samples/s
|
| 847 |
+
Step 22560/50462 | Loss: 2.6041 | LR: 2.98e-04 | 2737.4ms/step | 21.9 samples/s
|
| 848 |
+
Step 22570/50462 | Loss: 2.5891 | LR: 2.98e-04 | 2735.9ms/step | 21.9 samples/s
|
| 849 |
+
Step 22580/50462 | Loss: 2.6012 | LR: 2.98e-04 | 2742.9ms/step | 21.9 samples/s
|
| 850 |
+
Step 22590/50462 | Loss: 2.6363 | LR: 2.97e-04 | 2754.0ms/step | 21.8 samples/s
|
| 851 |
+
Step 22600/50462 | Loss: 2.6119 | LR: 2.97e-04 | 2734.2ms/step | 21.9 samples/s
|
| 852 |
+
Step 22610/50462 | Loss: 2.6029 | LR: 2.97e-04 | 2739.0ms/step | 21.9 samples/s
|
| 853 |
+
Step 22620/50462 | Loss: 2.6065 | LR: 2.97e-04 | 2740.4ms/step | 21.9 samples/s
|
| 854 |
+
Step 22630/50462 | Loss: 2.6299 | LR: 2.97e-04 | 2740.9ms/step | 21.9 samples/s
|
| 855 |
+
Step 22640/50462 | Loss: 2.6079 | LR: 2.97e-04 | 2744.1ms/step | 21.9 samples/s
|
| 856 |
+
Step 22650/50462 | Loss: 2.5897 | LR: 2.96e-04 | 2744.9ms/step | 21.9 samples/s
|
| 857 |
+
Step 22660/50462 | Loss: 2.5729 | LR: 2.96e-04 | 2740.9ms/step | 21.9 samples/s
|
| 858 |
+
Step 22670/50462 | Loss: 2.6158 | LR: 2.96e-04 | 2742.3ms/step | 21.9 samples/s
|
| 859 |
+
Step 22680/50462 | Loss: 2.6035 | LR: 2.96e-04 | 2740.4ms/step | 21.9 samples/s
|
| 860 |
+
Step 22690/50462 | Loss: 2.6226 | LR: 2.96e-04 | 2734.1ms/step | 21.9 samples/s
|
| 861 |
+
Step 22700/50462 | Loss: 2.6194 | LR: 2.96e-04 | 2736.3ms/step | 21.9 samples/s
|
| 862 |
+
Step 22710/50462 | Loss: 2.5782 | LR: 2.95e-04 | 2746.3ms/step | 21.8 samples/s
|
| 863 |
+
Step 22720/50462 | Loss: 2.6135 | LR: 2.95e-04 | 2741.0ms/step | 21.9 samples/s
|
| 864 |
+
Step 22730/50462 | Loss: 2.6448 | LR: 2.95e-04 | 2743.8ms/step | 21.9 samples/s
|
| 865 |
+
Step 22740/50462 | Loss: 2.6200 | LR: 2.95e-04 | 2746.7ms/step | 21.8 samples/s
|
| 866 |
+
Step 22750/50462 | Loss: 2.5823 | LR: 2.95e-04 | 2747.4ms/step | 21.8 samples/s
|
| 867 |
+
Validation at step 22750: Loss = 2.6145, Perplexity = 13.66
|
| 868 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-22750
|
| 869 |
+
Step 22760/50462 | Loss: 2.6007 | LR: 2.95e-04 | 4112.2ms/step | 14.6 samples/s
|
| 870 |
+
Step 22770/50462 | Loss: 2.6264 | LR: 2.95e-04 | 2741.5ms/step | 21.9 samples/s
|
| 871 |
+
Step 22780/50462 | Loss: 2.5848 | LR: 2.94e-04 | 2745.3ms/step | 21.9 samples/s
|
| 872 |
+
Step 22790/50462 | Loss: 2.6549 | LR: 2.94e-04 | 2744.8ms/step | 21.9 samples/s
|
| 873 |
+
Step 22800/50462 | Loss: 2.6285 | LR: 2.94e-04 | 2745.7ms/step | 21.9 samples/s
|
| 874 |
+
Step 22810/50462 | Loss: 2.6283 | LR: 2.94e-04 | 2750.0ms/step | 21.8 samples/s
|
| 875 |
+
Step 22820/50462 | Loss: 2.6155 | LR: 2.94e-04 | 2740.1ms/step | 21.9 samples/s
|
| 876 |
+
Step 22830/50462 | Loss: 2.5889 | LR: 2.94e-04 | 2736.3ms/step | 21.9 samples/s
|
| 877 |
+
Step 22840/50462 | Loss: 2.5858 | LR: 2.93e-04 | 2737.7ms/step | 21.9 samples/s
|
| 878 |
+
Step 22850/50462 | Loss: 2.6092 | LR: 2.93e-04 | 2742.9ms/step | 21.9 samples/s
|
| 879 |
+
Step 22860/50462 | Loss: 2.6140 | LR: 2.93e-04 | 2754.7ms/step | 21.8 samples/s
|
| 880 |
+
Step 22870/50462 | Loss: 2.6294 | LR: 2.93e-04 | 2745.6ms/step | 21.9 samples/s
|
| 881 |
+
Step 22880/50462 | Loss: 2.6232 | LR: 2.93e-04 | 2750.4ms/step | 21.8 samples/s
|
| 882 |
+
Step 22890/50462 | Loss: 2.6337 | LR: 2.93e-04 | 2744.0ms/step | 21.9 samples/s
|
| 883 |
+
Step 22900/50462 | Loss: 2.6094 | LR: 2.93e-04 | 2730.9ms/step | 22.0 samples/s
|
| 884 |
+
Step 22910/50462 | Loss: 2.5728 | LR: 2.92e-04 | 2739.8ms/step | 21.9 samples/s
|
| 885 |
+
Step 22920/50462 | Loss: 2.6224 | LR: 2.92e-04 | 2739.6ms/step | 21.9 samples/s
|
| 886 |
+
Step 22930/50462 | Loss: 2.5790 | LR: 2.92e-04 | 2739.6ms/step | 21.9 samples/s
|
| 887 |
+
Step 22940/50462 | Loss: 2.5862 | LR: 2.92e-04 | 2739.8ms/step | 21.9 samples/s
|
| 888 |
+
Step 22950/50462 | Loss: 2.5841 | LR: 2.92e-04 | 2740.8ms/step | 21.9 samples/s
|
| 889 |
+
Step 22960/50462 | Loss: 2.6266 | LR: 2.92e-04 | 2745.7ms/step | 21.9 samples/s
|
| 890 |
+
Step 22970/50462 | Loss: 2.6343 | LR: 2.91e-04 | 2742.9ms/step | 21.9 samples/s
|
| 891 |
+
Step 22980/50462 | Loss: 2.5959 | LR: 2.91e-04 | 2740.1ms/step | 21.9 samples/s
|
| 892 |
+
Step 22990/50462 | Loss: 2.6089 | LR: 2.91e-04 | 2746.6ms/step | 21.8 samples/s
|
| 893 |
+
Step 23000/50462 | Loss: 2.6103 | LR: 2.91e-04 | 2744.9ms/step | 21.9 samples/s
|
| 894 |
+
Validation at step 23000: Loss = 2.6121, Perplexity = 13.63
|
| 895 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23000
|
| 896 |
+
Step 23010/50462 | Loss: 2.6212 | LR: 2.91e-04 | 4115.5ms/step | 14.6 samples/s
|
| 897 |
+
Step 23020/50462 | Loss: 2.6052 | LR: 2.91e-04 | 2737.7ms/step | 21.9 samples/s
|
| 898 |
+
Step 23030/50462 | Loss: 2.6464 | LR: 2.91e-04 | 2741.6ms/step | 21.9 samples/s
|
| 899 |
+
Step 23040/50462 | Loss: 2.6411 | LR: 2.90e-04 | 2741.9ms/step | 21.9 samples/s
|
| 900 |
+
Step 23050/50462 | Loss: 2.6562 | LR: 2.90e-04 | 2742.7ms/step | 21.9 samples/s
|
| 901 |
+
Step 23060/50462 | Loss: 2.6161 | LR: 2.90e-04 | 2741.7ms/step | 21.9 samples/s
|
| 902 |
+
Step 23070/50462 | Loss: 2.6619 | LR: 2.90e-04 | 2743.1ms/step | 21.9 samples/s
|
| 903 |
+
Step 23080/50462 | Loss: 2.5846 | LR: 2.90e-04 | 2736.7ms/step | 21.9 samples/s
|
| 904 |
+
Step 23090/50462 | Loss: 2.6543 | LR: 2.90e-04 | 2733.5ms/step | 21.9 samples/s
|
| 905 |
+
Step 23100/50462 | Loss: 2.6152 | LR: 2.89e-04 | 2744.6ms/step | 21.9 samples/s
|
| 906 |
+
Step 23110/50462 | Loss: 2.6465 | LR: 2.89e-04 | 2739.2ms/step | 21.9 samples/s
|
| 907 |
+
Step 23120/50462 | Loss: 2.6165 | LR: 2.89e-04 | 2741.0ms/step | 21.9 samples/s
|
| 908 |
+
Step 23130/50462 | Loss: 2.6331 | LR: 2.89e-04 | 2737.0ms/step | 21.9 samples/s
|
| 909 |
+
Step 23140/50462 | Loss: 2.5907 | LR: 2.89e-04 | 2743.8ms/step | 21.9 samples/s
|
| 910 |
+
Step 23150/50462 | Loss: 2.6287 | LR: 2.89e-04 | 2732.1ms/step | 22.0 samples/s
|
| 911 |
+
Step 23160/50462 | Loss: 2.5863 | LR: 2.88e-04 | 2736.8ms/step | 21.9 samples/s
|
| 912 |
+
Step 23170/50462 | Loss: 2.6287 | LR: 2.88e-04 | 2745.9ms/step | 21.9 samples/s
|
| 913 |
+
Step 23180/50462 | Loss: 2.6383 | LR: 2.88e-04 | 2743.9ms/step | 21.9 samples/s
|
| 914 |
+
Step 23190/50462 | Loss: 2.6273 | LR: 2.88e-04 | 2742.0ms/step | 21.9 samples/s
|
| 915 |
+
Step 23200/50462 | Loss: 2.6217 | LR: 2.88e-04 | 2740.5ms/step | 21.9 samples/s
|
| 916 |
+
Step 23210/50462 | Loss: 2.6581 | LR: 2.88e-04 | 2742.1ms/step | 21.9 samples/s
|
| 917 |
+
Step 23220/50462 | Loss: 2.5811 | LR: 2.88e-04 | 2742.0ms/step | 21.9 samples/s
|
| 918 |
+
Step 23230/50462 | Loss: 2.6327 | LR: 2.87e-04 | 2743.3ms/step | 21.9 samples/s
|
| 919 |
+
Step 23240/50462 | Loss: 2.6386 | LR: 2.87e-04 | 2735.2ms/step | 21.9 samples/s
|
| 920 |
+
Step 23250/50462 | Loss: 2.6031 | LR: 2.87e-04 | 2741.1ms/step | 21.9 samples/s
|
| 921 |
+
Validation at step 23250: Loss = 2.6092, Perplexity = 13.59
|
| 922 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23250
|
| 923 |
+
Step 23260/50462 | Loss: 2.5776 | LR: 2.87e-04 | 4169.9ms/step | 14.4 samples/s
|
| 924 |
+
Step 23270/50462 | Loss: 2.5957 | LR: 2.87e-04 | 2740.0ms/step | 21.9 samples/s
|
| 925 |
+
Step 23280/50462 | Loss: 2.6194 | LR: 2.87e-04 | 2731.7ms/step | 22.0 samples/s
|
| 926 |
+
Step 23290/50462 | Loss: 2.5862 | LR: 2.86e-04 | 2756.4ms/step | 21.8 samples/s
|
| 927 |
+
Step 23300/50462 | Loss: 2.6336 | LR: 2.86e-04 | 2749.0ms/step | 21.8 samples/s
|
| 928 |
+
Step 23310/50462 | Loss: 2.5884 | LR: 2.86e-04 | 2746.6ms/step | 21.8 samples/s
|
| 929 |
+
Step 23320/50462 | Loss: 2.6186 | LR: 2.86e-04 | 2738.6ms/step | 21.9 samples/s
|
| 930 |
+
Step 23330/50462 | Loss: 2.6024 | LR: 2.86e-04 | 2736.4ms/step | 21.9 samples/s
|
| 931 |
+
Step 23340/50462 | Loss: 2.6323 | LR: 2.86e-04 | 2738.8ms/step | 21.9 samples/s
|
| 932 |
+
Step 23350/50462 | Loss: 2.6365 | LR: 2.86e-04 | 2737.1ms/step | 21.9 samples/s
|
| 933 |
+
Step 23360/50462 | Loss: 2.5711 | LR: 2.85e-04 | 2738.9ms/step | 21.9 samples/s
|
| 934 |
+
Step 23370/50462 | Loss: 2.6149 | LR: 2.85e-04 | 2742.8ms/step | 21.9 samples/s
|
| 935 |
+
Step 23380/50462 | Loss: 2.6502 | LR: 2.85e-04 | 2746.9ms/step | 21.8 samples/s
|
| 936 |
+
Step 23390/50462 | Loss: 2.6158 | LR: 2.85e-04 | 2737.3ms/step | 21.9 samples/s
|
| 937 |
+
Step 23400/50462 | Loss: 2.6128 | LR: 2.85e-04 | 2737.3ms/step | 21.9 samples/s
|
| 938 |
+
Step 23410/50462 | Loss: 2.6459 | LR: 2.85e-04 | 2733.5ms/step | 22.0 samples/s
|
| 939 |
+
Step 23420/50462 | Loss: 2.5980 | LR: 2.84e-04 | 2741.8ms/step | 21.9 samples/s
|
| 940 |
+
Step 23430/50462 | Loss: 2.5731 | LR: 2.84e-04 | 2742.1ms/step | 21.9 samples/s
|
| 941 |
+
Step 23440/50462 | Loss: 2.6093 | LR: 2.84e-04 | 2738.6ms/step | 21.9 samples/s
|
| 942 |
+
Step 23450/50462 | Loss: 2.5845 | LR: 2.84e-04 | 2743.2ms/step | 21.9 samples/s
|
| 943 |
+
Step 23460/50462 | Loss: 2.6291 | LR: 2.84e-04 | 2751.2ms/step | 21.8 samples/s
|
| 944 |
+
Step 23470/50462 | Loss: 2.6139 | LR: 2.84e-04 | 2746.3ms/step | 21.8 samples/s
|
| 945 |
+
Step 23480/50462 | Loss: 2.6379 | LR: 2.83e-04 | 2751.6ms/step | 21.8 samples/s
|
| 946 |
+
Step 23490/50462 | Loss: 2.5938 | LR: 2.83e-04 | 2735.7ms/step | 21.9 samples/s
|
| 947 |
+
Step 23500/50462 | Loss: 2.6277 | LR: 2.83e-04 | 2738.0ms/step | 21.9 samples/s
|
| 948 |
+
Validation at step 23500: Loss = 2.6078, Perplexity = 13.57
|
| 949 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23500
|
| 950 |
+
Step 23510/50462 | Loss: 2.5748 | LR: 2.83e-04 | 4114.6ms/step | 14.6 samples/s
|
| 951 |
+
Step 23520/50462 | Loss: 2.6445 | LR: 2.83e-04 | 2739.0ms/step | 21.9 samples/s
|
| 952 |
+
Step 23530/50462 | Loss: 2.5860 | LR: 2.83e-04 | 2738.7ms/step | 21.9 samples/s
|
| 953 |
+
Step 23540/50462 | Loss: 2.6137 | LR: 2.83e-04 | 2742.8ms/step | 21.9 samples/s
|
| 954 |
+
Step 23550/50462 | Loss: 2.5810 | LR: 2.82e-04 | 2749.6ms/step | 21.8 samples/s
|
| 955 |
+
Step 23560/50462 | Loss: 2.6400 | LR: 2.82e-04 | 2738.9ms/step | 21.9 samples/s
|
| 956 |
+
Step 23570/50462 | Loss: 2.6111 | LR: 2.82e-04 | 2737.1ms/step | 21.9 samples/s
|
| 957 |
+
Step 23580/50462 | Loss: 2.5847 | LR: 2.82e-04 | 2730.3ms/step | 22.0 samples/s
|
| 958 |
+
Step 23590/50462 | Loss: 2.5888 | LR: 2.82e-04 | 2743.6ms/step | 21.9 samples/s
|
| 959 |
+
Step 23600/50462 | Loss: 2.6120 | LR: 2.82e-04 | 2737.6ms/step | 21.9 samples/s
|
| 960 |
+
Step 23610/50462 | Loss: 2.5768 | LR: 2.81e-04 | 2734.3ms/step | 21.9 samples/s
|
| 961 |
+
Step 23620/50462 | Loss: 2.5928 | LR: 2.81e-04 | 2737.0ms/step | 21.9 samples/s
|
| 962 |
+
Step 23630/50462 | Loss: 2.6110 | LR: 2.81e-04 | 2743.6ms/step | 21.9 samples/s
|
| 963 |
+
Step 23640/50462 | Loss: 2.5981 | LR: 2.81e-04 | 2749.6ms/step | 21.8 samples/s
|
| 964 |
+
Step 23650/50462 | Loss: 2.6090 | LR: 2.81e-04 | 2739.7ms/step | 21.9 samples/s
|
| 965 |
+
Step 23660/50462 | Loss: 2.6489 | LR: 2.81e-04 | 2743.8ms/step | 21.9 samples/s
|
| 966 |
+
Step 23670/50462 | Loss: 2.6170 | LR: 2.81e-04 | 2742.0ms/step | 21.9 samples/s
|
| 967 |
+
Step 23680/50462 | Loss: 2.6174 | LR: 2.80e-04 | 2741.0ms/step | 21.9 samples/s
|
| 968 |
+
Step 23690/50462 | Loss: 2.5832 | LR: 2.80e-04 | 2738.0ms/step | 21.9 samples/s
|
| 969 |
+
Step 23700/50462 | Loss: 2.5772 | LR: 2.80e-04 | 2742.5ms/step | 21.9 samples/s
|
| 970 |
+
Step 23710/50462 | Loss: 2.6330 | LR: 2.80e-04 | 2732.0ms/step | 22.0 samples/s
|
| 971 |
+
Step 23720/50462 | Loss: 2.6028 | LR: 2.80e-04 | 2739.2ms/step | 21.9 samples/s
|
| 972 |
+
Step 23730/50462 | Loss: 2.5954 | LR: 2.80e-04 | 2742.8ms/step | 21.9 samples/s
|
| 973 |
+
Step 23740/50462 | Loss: 2.6088 | LR: 2.79e-04 | 2734.0ms/step | 21.9 samples/s
|
| 974 |
+
Step 23750/50462 | Loss: 2.5849 | LR: 2.79e-04 | 2728.4ms/step | 22.0 samples/s
|
| 975 |
+
Validation at step 23750: Loss = 2.6059, Perplexity = 13.54
|
| 976 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-23750
|
| 977 |
+
Step 23760/50462 | Loss: 2.6031 | LR: 2.79e-04 | 4113.1ms/step | 14.6 samples/s
|
| 978 |
+
Step 23770/50462 | Loss: 2.5934 | LR: 2.79e-04 | 2745.8ms/step | 21.9 samples/s
|
| 979 |
+
Step 23780/50462 | Loss: 2.5997 | LR: 2.79e-04 | 2731.7ms/step | 22.0 samples/s
|
| 980 |
+
Step 23790/50462 | Loss: 2.5823 | LR: 2.79e-04 | 2736.9ms/step | 21.9 samples/s
|
| 981 |
+
Step 23800/50462 | Loss: 2.5863 | LR: 2.78e-04 | 2741.3ms/step | 21.9 samples/s
|
| 982 |
+
Step 23810/50462 | Loss: 2.6289 | LR: 2.78e-04 | 2740.9ms/step | 21.9 samples/s
|
| 983 |
+
Step 23820/50462 | Loss: 2.5654 | LR: 2.78e-04 | 2735.9ms/step | 21.9 samples/s
|
| 984 |
+
Step 23830/50462 | Loss: 2.6132 | LR: 2.78e-04 | 2734.7ms/step | 21.9 samples/s
|
| 985 |
+
Step 23840/50462 | Loss: 2.6484 | LR: 2.78e-04 | 2735.8ms/step | 21.9 samples/s
|
| 986 |
+
Step 23850/50462 | Loss: 2.6028 | LR: 2.78e-04 | 2733.5ms/step | 22.0 samples/s
|
| 987 |
+
Step 23860/50462 | Loss: 2.5838 | LR: 2.78e-04 | 2740.0ms/step | 21.9 samples/s
|
| 988 |
+
Step 23870/50462 | Loss: 2.6412 | LR: 2.77e-04 | 2742.1ms/step | 21.9 samples/s
|
| 989 |
+
Step 23880/50462 | Loss: 2.5837 | LR: 2.77e-04 | 2736.8ms/step | 21.9 samples/s
|
| 990 |
+
Step 23890/50462 | Loss: 2.6055 | LR: 2.77e-04 | 2750.5ms/step | 21.8 samples/s
|
| 991 |
+
Step 23900/50462 | Loss: 2.6182 | LR: 2.77e-04 | 2735.5ms/step | 21.9 samples/s
|
| 992 |
+
Step 23910/50462 | Loss: 2.6253 | LR: 2.77e-04 | 2740.5ms/step | 21.9 samples/s
|
| 993 |
+
Step 23920/50462 | Loss: 2.5954 | LR: 2.77e-04 | 2740.6ms/step | 21.9 samples/s
|
| 994 |
+
Step 23930/50462 | Loss: 2.6062 | LR: 2.76e-04 | 2731.2ms/step | 22.0 samples/s
|
| 995 |
+
Step 23940/50462 | Loss: 2.5865 | LR: 2.76e-04 | 2730.1ms/step | 22.0 samples/s
|
| 996 |
+
Step 23950/50462 | Loss: 2.6170 | LR: 2.76e-04 | 2733.4ms/step | 22.0 samples/s
|
| 997 |
+
Step 23960/50462 | Loss: 2.5965 | LR: 2.76e-04 | 2742.2ms/step | 21.9 samples/s
|
| 998 |
+
Step 23970/50462 | Loss: 2.6080 | LR: 2.76e-04 | 2735.2ms/step | 21.9 samples/s
|
| 999 |
+
Step 23980/50462 | Loss: 2.6029 | LR: 2.76e-04 | 2743.7ms/step | 21.9 samples/s
|
| 1000 |
+
Step 23990/50462 | Loss: 2.6038 | LR: 2.75e-04 | 2740.5ms/step | 21.9 samples/s
|
| 1001 |
+
Step 24000/50462 | Loss: 2.5570 | LR: 2.75e-04 | 2746.2ms/step | 21.8 samples/s
|
| 1002 |
+
Validation at step 24000: Loss = 2.6042, Perplexity = 13.52
|
| 1003 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-24000
|
| 1004 |
+
Step 24010/50462 | Loss: 2.6207 | LR: 2.75e-04 | 4173.8ms/step | 14.4 samples/s
|
| 1005 |
+
Step 24020/50462 | Loss: 2.6265 | LR: 2.75e-04 | 2746.3ms/step | 21.8 samples/s
|
| 1006 |
+
Step 24030/50462 | Loss: 2.6250 | LR: 2.75e-04 | 2738.6ms/step | 21.9 samples/s
|
| 1007 |
+
Step 24040/50462 | Loss: 2.6039 | LR: 2.75e-04 | 2744.7ms/step | 21.9 samples/s
|
| 1008 |
+
Step 24050/50462 | Loss: 2.5788 | LR: 2.75e-04 | 2731.6ms/step | 22.0 samples/s
|
| 1009 |
+
Step 24060/50462 | Loss: 2.6087 | LR: 2.74e-04 | 2742.3ms/step | 21.9 samples/s
|
| 1010 |
+
Step 24070/50462 | Loss: 2.6095 | LR: 2.74e-04 | 2747.0ms/step | 21.8 samples/s
|
| 1011 |
+
Step 24080/50462 | Loss: 2.5971 | LR: 2.74e-04 | 2742.9ms/step | 21.9 samples/s
|
| 1012 |
+
Step 24090/50462 | Loss: 2.5383 | LR: 2.74e-04 | 2733.9ms/step | 21.9 samples/s
|
| 1013 |
+
Step 24100/50462 | Loss: 2.5782 | LR: 2.74e-04 | 2731.5ms/step | 22.0 samples/s
|
| 1014 |
+
Step 24110/50462 | Loss: 2.6012 | LR: 2.74e-04 | 2732.9ms/step | 22.0 samples/s
|
| 1015 |
+
Step 24120/50462 | Loss: 2.6129 | LR: 2.73e-04 | 2734.5ms/step | 21.9 samples/s
|
| 1016 |
+
Step 24130/50462 | Loss: 2.5750 | LR: 2.73e-04 | 2726.4ms/step | 22.0 samples/s
|
| 1017 |
+
Step 24140/50462 | Loss: 2.5965 | LR: 2.73e-04 | 2749.8ms/step | 21.8 samples/s
|
| 1018 |
+
Step 24150/50462 | Loss: 2.5781 | LR: 2.73e-04 | 2740.7ms/step | 21.9 samples/s
|
| 1019 |
+
Step 24160/50462 | Loss: 2.5982 | LR: 2.73e-04 | 2744.8ms/step | 21.9 samples/s
|
| 1020 |
+
Step 24170/50462 | Loss: 2.6245 | LR: 2.73e-04 | 2745.4ms/step | 21.9 samples/s
|
| 1021 |
+
Step 24180/50462 | Loss: 2.6082 | LR: 2.72e-04 | 2736.0ms/step | 21.9 samples/s
|
| 1022 |
+
Step 24190/50462 | Loss: 2.6170 | LR: 2.72e-04 | 2744.4ms/step | 21.9 samples/s
|
| 1023 |
+
Step 24200/50462 | Loss: 2.5987 | LR: 2.72e-04 | 2736.5ms/step | 21.9 samples/s
|
| 1024 |
+
Step 24210/50462 | Loss: 2.5777 | LR: 2.72e-04 | 2747.2ms/step | 21.8 samples/s
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch2.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_relaunch3.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed70_6gpu_resume48750.log
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
W0826 15:25:53.888000 4261 site-packages/torch/distributed/run.py:851]
|
| 2 |
+
W0826 15:25:53.888000 4261 site-packages/torch/distributed/run.py:851] *****************************************
|
| 3 |
+
W0826 15:25:53.888000 4261 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
W0826 15:25:53.888000 4261 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107
|
| 6 |
+
Running config:
|
| 7 |
+
{
|
| 8 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 9 |
+
"model_dtype": "bfloat16",
|
| 10 |
+
"init_from_scratch": true,
|
| 11 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 12 |
+
"val_path": null,
|
| 13 |
+
"max_seq_length": 8192,
|
| 14 |
+
"num_workers": 8,
|
| 15 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107",
|
| 16 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48750",
|
| 17 |
+
"force_new_run": false,
|
| 18 |
+
"num_epochs": 1,
|
| 19 |
+
"max_steps": 50462,
|
| 20 |
+
"per_device_batch_size": 1,
|
| 21 |
+
"gradient_accumulation_steps": 10,
|
| 22 |
+
"learning_rate": 0.0005,
|
| 23 |
+
"weight_decay": 0.01,
|
| 24 |
+
"warmup_steps": 750,
|
| 25 |
+
"lr_scheduler": "cosine",
|
| 26 |
+
"max_grad_norm": 1.0,
|
| 27 |
+
"save_steps": 250,
|
| 28 |
+
"eval_steps": 250,
|
| 29 |
+
"eval_batches": 50,
|
| 30 |
+
"logging_steps": 10,
|
| 31 |
+
"seed": 70,
|
| 32 |
+
"verbose": true,
|
| 33 |
+
"checkpoint_activations": true,
|
| 34 |
+
"strategy": "ddp",
|
| 35 |
+
"mixed_precision": "bf16",
|
| 36 |
+
"wandb_enabled": true,
|
| 37 |
+
"wandb_project": "stochastic_sparsity",
|
| 38 |
+
"wandb_name": "1.2b mixed 8k seed70 6gpu resume48750",
|
| 39 |
+
"wandb_tags": [
|
| 40 |
+
"qwen3",
|
| 41 |
+
"1.2b",
|
| 42 |
+
"8k",
|
| 43 |
+
"p2",
|
| 44 |
+
"mixed",
|
| 45 |
+
"seed70"
|
| 46 |
+
],
|
| 47 |
+
"hidden_size": 1536,
|
| 48 |
+
"intermediate_size": 6144,
|
| 49 |
+
"num_attention_heads": 12,
|
| 50 |
+
"num_key_value_heads": 4,
|
| 51 |
+
"num_hidden_layers": null,
|
| 52 |
+
"head_dim": null,
|
| 53 |
+
"attention_dropout": null,
|
| 54 |
+
"rope_train_position_scale_set": null,
|
| 55 |
+
"attention_type": "sparse_stochastic",
|
| 56 |
+
"sparse_block_size": 128,
|
| 57 |
+
"sparse_num_random_blocks": 35,
|
| 58 |
+
"sparse_mask_pool_size": 10000,
|
| 59 |
+
"sparse_val_dense_compare": false,
|
| 60 |
+
"sparse_layer_indices": null,
|
| 61 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 62 |
+
"sparse_mask_seed": 70,
|
| 63 |
+
"sparse_mixed_nrb_set": [
|
| 64 |
+
63,
|
| 65 |
+
47,
|
| 66 |
+
31,
|
| 67 |
+
21,
|
| 68 |
+
11
|
| 69 |
+
],
|
| 70 |
+
"dense_mode": false,
|
| 71 |
+
"dense_step_intervals": null
|
| 72 |
+
}
|
| 73 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 74 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48750
|
| 75 |
+
Checkpoint activations: True
|
| 76 |
+
Checkpoint activations: True
|
| 77 |
+
Checkpoint activations: True
|
| 78 |
+
Checkpoint activations: True
|
| 79 |
+
Checkpoint activations: True
|
| 80 |
+
Checkpoint activations: True
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
================================================================================
|
| 84 |
+
MODEL ARCHITECTURE
|
| 85 |
+
================================================================================
|
| 86 |
+
Model config:
|
| 87 |
+
vocab_size: 151936
|
| 88 |
+
hidden_size: 1536
|
| 89 |
+
num_hidden_layers: 28
|
| 90 |
+
num_attention_heads: 12
|
| 91 |
+
num_key_value_heads: 4
|
| 92 |
+
head_dim: 128
|
| 93 |
+
intermediate_size: 6144
|
| 94 |
+
max_position_embeddings: 32768
|
| 95 |
+
rope_theta: 1000000
|
| 96 |
+
rope_train_position_scale_set: None
|
| 97 |
+
attention_type: sparse_stochastic
|
| 98 |
+
sparse_block_size: 128
|
| 99 |
+
sparse_num_random_blocks: 35
|
| 100 |
+
sparse_mask_pool_size: 10000
|
| 101 |
+
sparse_max_seq_length: 8192
|
| 102 |
+
|
| 103 |
+
Parameter count:
|
| 104 |
+
Total parameters: 1,202,352,640
|
| 105 |
+
Trainable parameters: 1,202,352,640
|
| 106 |
+
Non-trainable parameters: 0
|
| 107 |
+
Model size (BF16): 2.40 GB
|
| 108 |
+
|
| 109 |
+
--------------------------------------------------------------------------------
|
| 110 |
+
LAYER-BY-LAYER SUMMARY
|
| 111 |
+
--------------------------------------------------------------------------------
|
| 112 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 113 |
+
|
| 114 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 115 |
+
|
| 116 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 117 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 118 |
+
|
| 119 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 120 |
+
_warn_once(
|
| 121 |
+
================================================================================================================================================================
|
| 122 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 123 |
+
================================================================================================================================================================
|
| 124 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 125 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 126 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 127 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 153 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 154 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 155 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 156 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 157 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 158 |
+
================================================================================================================================================================
|
| 159 |
+
Total params: 1,435,726,336
|
| 160 |
+
Trainable params: 1,435,726,336
|
| 161 |
+
Non-trainable params: 0
|
| 162 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 163 |
+
================================================================================================================================================================
|
| 164 |
+
Input size (MB): 0.07
|
| 165 |
+
Forward/backward pass size (MB): 26218.59
|
| 166 |
+
Params size (MB): 2871.45
|
| 167 |
+
Estimated Total Size (MB): 29090.11
|
| 168 |
+
================================================================================================================================================================
|
| 169 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 170 |
+
================================================================================
|
| 171 |
+
|
| 172 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 173 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 174 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 175 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 176 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 177 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 178 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 179 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 180 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 181 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 182 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 183 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 184 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 185 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 186 |
+
Dataset split: 3027764 train, 159356 validation
|
| 187 |
+
Training examples (packed, train split): 3027764
|
| 188 |
+
Validation examples (packed, val split): 159356
|
| 189 |
+
Per-device batch size: 1
|
| 190 |
+
World size (GPUs): 6
|
| 191 |
+
Gradient accumulation steps: 10
|
| 192 |
+
Effective batch size (seqs/step): 60
|
| 193 |
+
Sequence length: 8192
|
| 194 |
+
Tokens per optimizer step: 491,520
|
| 195 |
+
Micro-batches per epoch (per rank): 504628
|
| 196 |
+
Optimizer steps per epoch: 50462
|
| 197 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 198 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48750...
|
| 199 |
+
Preserving checkpoint LR state for exact resume: current=[1.4617433771005884e-06], schedule_base=[0.0005]
|
| 200 |
+
Resumed from checkpoint at step 48750 (epoch 0, micro_step 502500)
|
| 201 |
+
Resuming training from step 48750
|
| 202 |
+
Resuming wandb run i8ewyy95
|
| 203 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 204 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 205 |
+
wandb: setting up run i8ewyy95
|
| 206 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 207 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260826_152940-i8ewyy95
|
| 208 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 209 |
+
wandb: Resuming run 1.2b mixed 8k seed70 6gpu resume48750
|
| 210 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 211 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i8ewyy95
|
| 212 |
+
Starting training...
|
| 213 |
+
Validation at step 48750: Loss = 2.5281, Perplexity = 12.53
|
| 214 |
+
wandb: WARNING Tried to log to step 48750 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Switching attention mode for step 48751: sparse_stochastic (configured sparse attention)
|
| 216 |
+
Step 48760/50462 | Loss: 2.5039 | LR: 1.44e-06 | 74877.1ms/step | 0.8 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 48760 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 48770/50462 | Loss: 2.5169 | LR: 1.43e-06 | 3307.9ms/step | 18.1 samples/s
|
| 219 |
+
wandb: WARNING Tried to log to step 48770 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 220 |
+
Step 48780/50462 | Loss: 2.5173 | LR: 1.41e-06 | 2735.2ms/step | 21.9 samples/s
|
| 221 |
+
wandb: WARNING Tried to log to step 48780 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 222 |
+
Step 48790/50462 | Loss: 2.5569 | LR: 1.39e-06 | 2736.4ms/step | 21.9 samples/s
|
| 223 |
+
wandb: WARNING Tried to log to step 48790 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 224 |
+
Step 48800/50462 | Loss: 2.5348 | LR: 1.38e-06 | 2734.6ms/step | 21.9 samples/s
|
| 225 |
+
wandb: WARNING Tried to log to step 48800 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 226 |
+
Step 48810/50462 | Loss: 2.5282 | LR: 1.36e-06 | 2742.2ms/step | 21.9 samples/s
|
| 227 |
+
wandb: WARNING Tried to log to step 48810 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 228 |
+
Step 48820/50462 | Loss: 2.5347 | LR: 1.34e-06 | 2748.7ms/step | 21.8 samples/s
|
| 229 |
+
wandb: WARNING Tried to log to step 48820 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 230 |
+
Step 48830/50462 | Loss: 2.5142 | LR: 1.33e-06 | 2740.7ms/step | 21.9 samples/s
|
| 231 |
+
wandb: WARNING Tried to log to step 48830 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 232 |
+
Step 48840/50462 | Loss: 2.5337 | LR: 1.31e-06 | 2746.6ms/step | 21.8 samples/s
|
| 233 |
+
wandb: WARNING Tried to log to step 48840 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 234 |
+
Step 48850/50462 | Loss: 2.5308 | LR: 1.30e-06 | 2740.9ms/step | 21.9 samples/s
|
| 235 |
+
wandb: WARNING Tried to log to step 48850 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 236 |
+
Step 48860/50462 | Loss: 2.4891 | LR: 1.28e-06 | 2744.3ms/step | 21.9 samples/s
|
| 237 |
+
wandb: WARNING Tried to log to step 48860 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 238 |
+
Step 48870/50462 | Loss: 2.5272 | LR: 1.26e-06 | 2735.8ms/step | 21.9 samples/s
|
| 239 |
+
wandb: WARNING Tried to log to step 48870 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 240 |
+
Step 48880/50462 | Loss: 2.5142 | LR: 1.25e-06 | 2738.2ms/step | 21.9 samples/s
|
| 241 |
+
wandb: WARNING Tried to log to step 48880 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 242 |
+
Step 48890/50462 | Loss: 2.5033 | LR: 1.23e-06 | 2735.5ms/step | 21.9 samples/s
|
| 243 |
+
wandb: WARNING Tried to log to step 48890 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 244 |
+
Step 48900/50462 | Loss: 2.5143 | LR: 1.22e-06 | 3301.6ms/step | 18.2 samples/s
|
| 245 |
+
wandb: WARNING Tried to log to step 48900 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 246 |
+
Step 48910/50462 | Loss: 2.5073 | LR: 1.20e-06 | 2739.0ms/step | 21.9 samples/s
|
| 247 |
+
wandb: WARNING Tried to log to step 48910 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 248 |
+
Step 48920/50462 | Loss: 2.4855 | LR: 1.19e-06 | 2737.2ms/step | 21.9 samples/s
|
| 249 |
+
wandb: WARNING Tried to log to step 48920 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 250 |
+
Step 48930/50462 | Loss: 2.5544 | LR: 1.17e-06 | 2736.7ms/step | 21.9 samples/s
|
| 251 |
+
wandb: WARNING Tried to log to step 48930 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 252 |
+
Step 48940/50462 | Loss: 2.4489 | LR: 1.16e-06 | 2736.7ms/step | 21.9 samples/s
|
| 253 |
+
wandb: WARNING Tried to log to step 48940 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 254 |
+
Step 48950/50462 | Loss: 2.5202 | LR: 1.14e-06 | 2733.4ms/step | 22.0 samples/s
|
| 255 |
+
wandb: WARNING Tried to log to step 48950 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 256 |
+
Step 48960/50462 | Loss: 2.5298 | LR: 1.13e-06 | 2742.7ms/step | 21.9 samples/s
|
| 257 |
+
wandb: WARNING Tried to log to step 48960 that is less than the current step 48961. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 258 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed70/run_20260822_210107/checkpoint-48962
|
| 259 |
+
wandb: updating run metadata
|
| 260 |
+
wandb: uploading wandb-summary.json; uploading config.yaml
|
| 261 |
+
wandb: uploading summary, console lines 46-46
|
| 262 |
+
wandb:
|
| 263 |
+
wandb: Run summary:
|
| 264 |
+
wandb: eval/dense_interval_active 0
|
| 265 |
+
wandb: eval/dense_interval_end -1
|
| 266 |
+
wandb: eval/dense_interval_start -1
|
| 267 |
+
wandb: eval/loss 2.52811
|
| 268 |
+
wandb: eval/perplexity 12.52985
|
| 269 |
+
wandb: eval/uses_dense_attention 0
|
| 270 |
+
wandb: eval/uses_sparse_attention 1
|
| 271 |
+
wandb: speed/ms_per_step 2729.08297
|
| 272 |
+
wandb: speed/samples_per_sec 21.98541
|
| 273 |
+
wandb: speed/steps_per_sec 0.36642
|
| 274 |
+
wandb: +4 ...
|
| 275 |
+
wandb:
|
| 276 |
+
wandb: 🚀 View run 1.2b mixed 8k seed70 6gpu resume48750 at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/i8ewyy95
|
| 277 |
+
wandb: ⭐️ View project at: https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 278 |
+
wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
|
| 279 |
+
wandb: Find logs at: ./wandb/run-20260826_152940-i8ewyy95/logs
|
| 280 |
+
Training complete!
|
epfl_final/train_logs/p2_1p2b_mixed_seed71_4gpu.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed71_5gpu_autoresume.log
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
===== AUTO-RESUME FROM checkpoint-16500 2026-08-27T13:41:25Z =====
|
| 3 |
+
W0827 13:41:35.484000 165 site-packages/torch/distributed/run.py:851]
|
| 4 |
+
W0827 13:41:35.484000 165 site-packages/torch/distributed/run.py:851] *****************************************
|
| 5 |
+
W0827 13:41:35.484000 165 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 6 |
+
W0827 13:41:35.484000 165 site-packages/torch/distributed/run.py:851] *****************************************
|
| 7 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
|
| 8 |
+
Running config:
|
| 9 |
+
{
|
| 10 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 11 |
+
"model_dtype": "bfloat16",
|
| 12 |
+
"init_from_scratch": true,
|
| 13 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 14 |
+
"val_path": null,
|
| 15 |
+
"max_seq_length": 8192,
|
| 16 |
+
"num_workers": 8,
|
| 17 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
|
| 18 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500",
|
| 19 |
+
"force_new_run": false,
|
| 20 |
+
"num_epochs": 1,
|
| 21 |
+
"max_steps": 50462,
|
| 22 |
+
"per_device_batch_size": 1,
|
| 23 |
+
"gradient_accumulation_steps": 12,
|
| 24 |
+
"learning_rate": 0.0005,
|
| 25 |
+
"weight_decay": 0.01,
|
| 26 |
+
"warmup_steps": 750,
|
| 27 |
+
"lr_scheduler": "cosine",
|
| 28 |
+
"max_grad_norm": 1.0,
|
| 29 |
+
"save_steps": 250,
|
| 30 |
+
"eval_steps": 250,
|
| 31 |
+
"eval_batches": 50,
|
| 32 |
+
"logging_steps": 10,
|
| 33 |
+
"seed": 71,
|
| 34 |
+
"verbose": true,
|
| 35 |
+
"checkpoint_activations": true,
|
| 36 |
+
"strategy": "ddp",
|
| 37 |
+
"mixed_precision": "bf16",
|
| 38 |
+
"wandb_enabled": true,
|
| 39 |
+
"wandb_project": "stochastic_sparsity",
|
| 40 |
+
"wandb_name": "1.2b mixed 8k seed71 4gpu",
|
| 41 |
+
"wandb_tags": [
|
| 42 |
+
"qwen3",
|
| 43 |
+
"1.2b",
|
| 44 |
+
"8k",
|
| 45 |
+
"p2",
|
| 46 |
+
"mixed",
|
| 47 |
+
"seed71"
|
| 48 |
+
],
|
| 49 |
+
"hidden_size": 1536,
|
| 50 |
+
"intermediate_size": 6144,
|
| 51 |
+
"num_attention_heads": 12,
|
| 52 |
+
"num_key_value_heads": 4,
|
| 53 |
+
"num_hidden_layers": null,
|
| 54 |
+
"head_dim": null,
|
| 55 |
+
"attention_dropout": null,
|
| 56 |
+
"rope_train_position_scale_set": null,
|
| 57 |
+
"attention_type": "sparse_stochastic",
|
| 58 |
+
"sparse_block_size": 128,
|
| 59 |
+
"sparse_num_random_blocks": 35,
|
| 60 |
+
"sparse_mask_pool_size": 10000,
|
| 61 |
+
"sparse_val_dense_compare": false,
|
| 62 |
+
"sparse_layer_indices": null,
|
| 63 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 64 |
+
"sparse_mask_seed": 71,
|
| 65 |
+
"sparse_mixed_nrb_set": [
|
| 66 |
+
63,
|
| 67 |
+
47,
|
| 68 |
+
31,
|
| 69 |
+
21,
|
| 70 |
+
11
|
| 71 |
+
],
|
| 72 |
+
"dense_mode": false,
|
| 73 |
+
"dense_step_intervals": null
|
| 74 |
+
}
|
| 75 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 76 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500
|
| 77 |
+
Checkpoint activations: TrueCheckpoint activations: True
|
| 78 |
+
Checkpoint activations: True
|
| 79 |
+
|
| 80 |
+
Checkpoint activations: True
|
| 81 |
+
Checkpoint activations: True
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
================================================================================
|
| 85 |
+
MODEL ARCHITECTURE
|
| 86 |
+
================================================================================
|
| 87 |
+
Model config:
|
| 88 |
+
vocab_size: 151936
|
| 89 |
+
hidden_size: 1536
|
| 90 |
+
num_hidden_layers: 28
|
| 91 |
+
num_attention_heads: 12
|
| 92 |
+
num_key_value_heads: 4
|
| 93 |
+
head_dim: 128
|
| 94 |
+
intermediate_size: 6144
|
| 95 |
+
max_position_embeddings: 32768
|
| 96 |
+
rope_theta: 1000000
|
| 97 |
+
rope_train_position_scale_set: None
|
| 98 |
+
attention_type: sparse_stochastic
|
| 99 |
+
sparse_block_size: 128
|
| 100 |
+
sparse_num_random_blocks: 35
|
| 101 |
+
sparse_mask_pool_size: 10000
|
| 102 |
+
sparse_max_seq_length: 8192
|
| 103 |
+
|
| 104 |
+
Parameter count:
|
| 105 |
+
Total parameters: 1,202,352,640
|
| 106 |
+
Trainable parameters: 1,202,352,640
|
| 107 |
+
Non-trainable parameters: 0
|
| 108 |
+
Model size (BF16): 2.40 GB
|
| 109 |
+
|
| 110 |
+
--------------------------------------------------------------------------------
|
| 111 |
+
LAYER-BY-LAYER SUMMARY
|
| 112 |
+
--------------------------------------------------------------------------------
|
| 113 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 114 |
+
|
| 115 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 116 |
+
|
| 117 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 118 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 119 |
+
|
| 120 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 121 |
+
_warn_once(
|
| 122 |
+
================================================================================================================================================================
|
| 123 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 124 |
+
================================================================================================================================================================
|
| 125 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 126 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 127 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 128 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 153 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 154 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 155 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 156 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 157 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 158 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 159 |
+
================================================================================================================================================================
|
| 160 |
+
Total params: 1,435,726,336
|
| 161 |
+
Trainable params: 1,435,726,336
|
| 162 |
+
Non-trainable params: 0
|
| 163 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 164 |
+
================================================================================================================================================================
|
| 165 |
+
Input size (MB): 0.07
|
| 166 |
+
Forward/backward pass size (MB): 26218.59
|
| 167 |
+
Params size (MB): 2871.45
|
| 168 |
+
Estimated Total Size (MB): 29090.11
|
| 169 |
+
================================================================================================================================================================
|
| 170 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 171 |
+
================================================================================
|
| 172 |
+
|
| 173 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 174 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 175 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 176 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 177 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 178 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 179 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 180 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 184 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 185 |
+
Dataset split: 3027764 train, 159356 validation
|
| 186 |
+
Training examples (packed, train split): 3027764
|
| 187 |
+
Validation examples (packed, val split): 159356
|
| 188 |
+
Per-device batch size: 1
|
| 189 |
+
World size (GPUs): 5
|
| 190 |
+
Gradient accumulation steps: 12
|
| 191 |
+
Effective batch size (seqs/step): 60
|
| 192 |
+
Sequence length: 8192
|
| 193 |
+
Tokens per optimizer step: 491,520
|
| 194 |
+
Micro-batches per epoch (per rank): 605553
|
| 195 |
+
Optimizer steps per epoch: 50462
|
| 196 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 197 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500...
|
| 198 |
+
Preserving checkpoint LR state for exact resume: current=[0.0003860555075981444], schedule_base=[0.0005]
|
| 199 |
+
Resumed from checkpoint at step 16500 (epoch 0, micro_step 245000)
|
| 200 |
+
Resuming training from step 16500
|
| 201 |
+
Resuming wandb run 8ynn1we6
|
| 202 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 203 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 204 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 205 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_134507-8ynn1we6
|
| 206 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 207 |
+
wandb: Resuming run 1.2b mixed 8k seed71 4gpu
|
| 208 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 209 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
|
| 210 |
+
Starting training...
|
epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_autoresume.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed71_6gpu_swapped.log
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
===== SEED71_TO_6GPU_FROM_16000 2026-08-27T12:31:44Z =====
|
| 2 |
+
W0827 12:31:46.220000 13779 site-packages/torch/distributed/run.py:851]
|
| 3 |
+
W0827 12:31:46.220000 13779 site-packages/torch/distributed/run.py:851] *****************************************
|
| 4 |
+
W0827 12:31:46.220000 13779 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 5 |
+
W0827 12:31:46.220000 13779 site-packages/torch/distributed/run.py:851] *****************************************
|
| 6 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205
|
| 7 |
+
Running config:
|
| 8 |
+
{
|
| 9 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 10 |
+
"model_dtype": "bfloat16",
|
| 11 |
+
"init_from_scratch": true,
|
| 12 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 13 |
+
"val_path": null,
|
| 14 |
+
"max_seq_length": 8192,
|
| 15 |
+
"num_workers": 8,
|
| 16 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205",
|
| 17 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16000",
|
| 18 |
+
"force_new_run": false,
|
| 19 |
+
"num_epochs": 1,
|
| 20 |
+
"max_steps": 50462,
|
| 21 |
+
"per_device_batch_size": 1,
|
| 22 |
+
"gradient_accumulation_steps": 10,
|
| 23 |
+
"learning_rate": 0.0005,
|
| 24 |
+
"weight_decay": 0.01,
|
| 25 |
+
"warmup_steps": 750,
|
| 26 |
+
"lr_scheduler": "cosine",
|
| 27 |
+
"max_grad_norm": 1.0,
|
| 28 |
+
"save_steps": 250,
|
| 29 |
+
"eval_steps": 250,
|
| 30 |
+
"eval_batches": 50,
|
| 31 |
+
"logging_steps": 10,
|
| 32 |
+
"seed": 71,
|
| 33 |
+
"verbose": true,
|
| 34 |
+
"checkpoint_activations": true,
|
| 35 |
+
"strategy": "ddp",
|
| 36 |
+
"mixed_precision": "bf16",
|
| 37 |
+
"wandb_enabled": true,
|
| 38 |
+
"wandb_project": "stochastic_sparsity",
|
| 39 |
+
"wandb_name": "1.2b mixed 8k seed71 4gpu",
|
| 40 |
+
"wandb_tags": [
|
| 41 |
+
"qwen3",
|
| 42 |
+
"1.2b",
|
| 43 |
+
"8k",
|
| 44 |
+
"p2",
|
| 45 |
+
"mixed",
|
| 46 |
+
"seed71"
|
| 47 |
+
],
|
| 48 |
+
"hidden_size": 1536,
|
| 49 |
+
"intermediate_size": 6144,
|
| 50 |
+
"num_attention_heads": 12,
|
| 51 |
+
"num_key_value_heads": 4,
|
| 52 |
+
"num_hidden_layers": null,
|
| 53 |
+
"head_dim": null,
|
| 54 |
+
"attention_dropout": null,
|
| 55 |
+
"rope_train_position_scale_set": null,
|
| 56 |
+
"attention_type": "sparse_stochastic",
|
| 57 |
+
"sparse_block_size": 128,
|
| 58 |
+
"sparse_num_random_blocks": 35,
|
| 59 |
+
"sparse_mask_pool_size": 10000,
|
| 60 |
+
"sparse_val_dense_compare": false,
|
| 61 |
+
"sparse_layer_indices": null,
|
| 62 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 63 |
+
"sparse_mask_seed": 71,
|
| 64 |
+
"sparse_mixed_nrb_set": [
|
| 65 |
+
63,
|
| 66 |
+
47,
|
| 67 |
+
31,
|
| 68 |
+
21,
|
| 69 |
+
11
|
| 70 |
+
],
|
| 71 |
+
"dense_mode": false,
|
| 72 |
+
"dense_step_intervals": null
|
| 73 |
+
}
|
| 74 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 75 |
+
Checkpoint activations: True
|
| 76 |
+
Checkpoint activations: True
|
| 77 |
+
Checkpoint activations: True
|
| 78 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16000
|
| 79 |
+
Checkpoint activations: True
|
| 80 |
+
Checkpoint activations: True
|
| 81 |
+
Checkpoint activations: True
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
================================================================================
|
| 85 |
+
MODEL ARCHITECTURE
|
| 86 |
+
================================================================================
|
| 87 |
+
Model config:
|
| 88 |
+
vocab_size: 151936
|
| 89 |
+
hidden_size: 1536
|
| 90 |
+
num_hidden_layers: 28
|
| 91 |
+
num_attention_heads: 12
|
| 92 |
+
num_key_value_heads: 4
|
| 93 |
+
head_dim: 128
|
| 94 |
+
intermediate_size: 6144
|
| 95 |
+
max_position_embeddings: 32768
|
| 96 |
+
rope_theta: 1000000
|
| 97 |
+
rope_train_position_scale_set: None
|
| 98 |
+
attention_type: sparse_stochastic
|
| 99 |
+
sparse_block_size: 128
|
| 100 |
+
sparse_num_random_blocks: 35
|
| 101 |
+
sparse_mask_pool_size: 10000
|
| 102 |
+
sparse_max_seq_length: 8192
|
| 103 |
+
|
| 104 |
+
Parameter count:
|
| 105 |
+
Total parameters: 1,202,352,640
|
| 106 |
+
Trainable parameters: 1,202,352,640
|
| 107 |
+
Non-trainable parameters: 0
|
| 108 |
+
Model size (BF16): 2.40 GB
|
| 109 |
+
|
| 110 |
+
--------------------------------------------------------------------------------
|
| 111 |
+
LAYER-BY-LAYER SUMMARY
|
| 112 |
+
--------------------------------------------------------------------------------
|
| 113 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 114 |
+
|
| 115 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 116 |
+
|
| 117 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 118 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 119 |
+
|
| 120 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 121 |
+
_warn_once(
|
| 122 |
+
================================================================================================================================================================
|
| 123 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 124 |
+
================================================================================================================================================================
|
| 125 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 126 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 127 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 128 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 153 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 154 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 155 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 156 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 157 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 158 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 159 |
+
================================================================================================================================================================
|
| 160 |
+
Total params: 1,435,726,336
|
| 161 |
+
Trainable params: 1,435,726,336
|
| 162 |
+
Non-trainable params: 0
|
| 163 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 164 |
+
================================================================================================================================================================
|
| 165 |
+
Input size (MB): 0.07
|
| 166 |
+
Forward/backward pass size (MB): 26218.59
|
| 167 |
+
Params size (MB): 2871.45
|
| 168 |
+
Estimated Total Size (MB): 29090.11
|
| 169 |
+
================================================================================================================================================================
|
| 170 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 171 |
+
================================================================================
|
| 172 |
+
|
| 173 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 174 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 175 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 176 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 177 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 178 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 179 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 180 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 181 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 182 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 183 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 184 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 185 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 186 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 187 |
+
Dataset split: 3027764 train, 159356 validation
|
| 188 |
+
Training examples (packed, train split): 3027764
|
| 189 |
+
Validation examples (packed, val split): 159356
|
| 190 |
+
Per-device batch size: 1
|
| 191 |
+
World size (GPUs): 6
|
| 192 |
+
Gradient accumulation steps: 10
|
| 193 |
+
Effective batch size (seqs/step): 60
|
| 194 |
+
Sequence length: 8192
|
| 195 |
+
Tokens per optimizer step: 491,520
|
| 196 |
+
Micro-batches per epoch (per rank): 504628
|
| 197 |
+
Optimizer steps per epoch: 50462
|
| 198 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 199 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16000...
|
| 200 |
+
Preserving checkpoint LR state for exact resume: current=[0.00039261369127550097], schedule_base=[0.0005]
|
| 201 |
+
Resumed from checkpoint at step 16000 (epoch 0, micro_step 240000)
|
| 202 |
+
Resuming training from step 16000
|
| 203 |
+
Resuming wandb run 8ynn1we6
|
| 204 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 205 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 206 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 207 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_123421-8ynn1we6
|
| 208 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 209 |
+
wandb: Resuming run 1.2b mixed 8k seed71 4gpu
|
| 210 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 211 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/8ynn1we6
|
| 212 |
+
Starting training...
|
| 213 |
+
Validation at step 16000: Loss = 2.6750, Perplexity = 14.51
|
| 214 |
+
wandb: WARNING Tried to log to step 16000 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Switching attention mode for step 16001: sparse_stochastic (configured sparse attention)
|
| 216 |
+
Step 16010/50462 | Loss: 2.6873 | LR: 3.92e-04 | 33781.4ms/step | 1.8 samples/s
|
| 217 |
+
wandb: WARNING Tried to log to step 16010 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 218 |
+
Step 16020/50462 | Loss: 2.6630 | LR: 3.92e-04 | 2724.5ms/step | 22.0 samples/s
|
| 219 |
+
wandb: WARNING Tried to log to step 16020 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 220 |
+
Step 16030/50462 | Loss: 2.7200 | LR: 3.92e-04 | 2720.4ms/step | 22.1 samples/s
|
| 221 |
+
wandb: WARNING Tried to log to step 16030 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 222 |
+
Step 16040/50462 | Loss: 2.6245 | LR: 3.92e-04 | 2722.8ms/step | 22.0 samples/s
|
| 223 |
+
wandb: WARNING Tried to log to step 16040 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 224 |
+
Step 16050/50462 | Loss: 2.6888 | LR: 3.92e-04 | 2734.5ms/step | 21.9 samples/s
|
| 225 |
+
wandb: WARNING Tried to log to step 16050 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 226 |
+
Step 16060/50462 | Loss: 2.6672 | LR: 3.92e-04 | 2736.9ms/step | 21.9 samples/s
|
| 227 |
+
wandb: WARNING Tried to log to step 16060 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 228 |
+
Step 16070/50462 | Loss: 2.7080 | LR: 3.92e-04 | 2736.1ms/step | 21.9 samples/s
|
| 229 |
+
wandb: WARNING Tried to log to step 16070 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 230 |
+
Step 16080/50462 | Loss: 2.6757 | LR: 3.92e-04 | 2725.1ms/step | 22.0 samples/s
|
| 231 |
+
wandb: WARNING Tried to log to step 16080 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 232 |
+
Step 16090/50462 | Loss: 2.6632 | LR: 3.91e-04 | 2725.8ms/step | 22.0 samples/s
|
| 233 |
+
wandb: WARNING Tried to log to step 16090 that is less than the current step 16091. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 234 |
+
Step 16100/50462 | Loss: 2.7036 | LR: 3.91e-04 | 2727.2ms/step | 22.0 samples/s
|
| 235 |
+
Step 16110/50462 | Loss: 2.6840 | LR: 3.91e-04 | 2734.4ms/step | 21.9 samples/s
|
| 236 |
+
Step 16120/50462 | Loss: 2.7376 | LR: 3.91e-04 | 2722.7ms/step | 22.0 samples/s
|
| 237 |
+
Step 16130/50462 | Loss: 2.6690 | LR: 3.91e-04 | 2732.3ms/step | 22.0 samples/s
|
| 238 |
+
Step 16140/50462 | Loss: 2.6952 | LR: 3.91e-04 | 2728.3ms/step | 22.0 samples/s
|
| 239 |
+
Step 16150/50462 | Loss: 2.6696 | LR: 3.91e-04 | 2726.6ms/step | 22.0 samples/s
|
| 240 |
+
Step 16160/50462 | Loss: 2.6968 | LR: 3.91e-04 | 2732.1ms/step | 22.0 samples/s
|
| 241 |
+
Step 16170/50462 | Loss: 2.7272 | LR: 3.90e-04 | 2732.8ms/step | 22.0 samples/s
|
| 242 |
+
Step 16180/50462 | Loss: 2.6767 | LR: 3.90e-04 | 2730.1ms/step | 22.0 samples/s
|
| 243 |
+
Step 16190/50462 | Loss: 2.7063 | LR: 3.90e-04 | 2735.5ms/step | 21.9 samples/s
|
| 244 |
+
Step 16200/50462 | Loss: 2.6885 | LR: 3.90e-04 | 2726.1ms/step | 22.0 samples/s
|
| 245 |
+
Step 16210/50462 | Loss: 2.6728 | LR: 3.90e-04 | 2730.2ms/step | 22.0 samples/s
|
| 246 |
+
Step 16220/50462 | Loss: 2.7079 | LR: 3.90e-04 | 2723.3ms/step | 22.0 samples/s
|
| 247 |
+
Step 16230/50462 | Loss: 2.6810 | LR: 3.90e-04 | 2734.6ms/step | 21.9 samples/s
|
| 248 |
+
Step 16240/50462 | Loss: 2.6902 | LR: 3.89e-04 | 2734.2ms/step | 21.9 samples/s
|
| 249 |
+
Step 16250/50462 | Loss: 2.6469 | LR: 3.89e-04 | 2725.4ms/step | 22.0 samples/s
|
| 250 |
+
Validation at step 16250: Loss = 2.6694, Perplexity = 14.43
|
| 251 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16250
|
| 252 |
+
Step 16260/50462 | Loss: 2.6882 | LR: 3.89e-04 | 4106.3ms/step | 14.6 samples/s
|
| 253 |
+
Step 16270/50462 | Loss: 2.6932 | LR: 3.89e-04 | 2732.7ms/step | 22.0 samples/s
|
| 254 |
+
Step 16280/50462 | Loss: 2.7110 | LR: 3.89e-04 | 2729.1ms/step | 22.0 samples/s
|
| 255 |
+
Step 16290/50462 | Loss: 2.6770 | LR: 3.89e-04 | 2737.8ms/step | 21.9 samples/s
|
| 256 |
+
Step 16300/50462 | Loss: 2.6872 | LR: 3.89e-04 | 2729.0ms/step | 22.0 samples/s
|
| 257 |
+
Step 16310/50462 | Loss: 2.7363 | LR: 3.89e-04 | 2732.5ms/step | 22.0 samples/s
|
| 258 |
+
Step 16320/50462 | Loss: 2.6883 | LR: 3.88e-04 | 2733.7ms/step | 21.9 samples/s
|
| 259 |
+
Step 16330/50462 | Loss: 2.6621 | LR: 3.88e-04 | 2731.7ms/step | 22.0 samples/s
|
| 260 |
+
Step 16340/50462 | Loss: 2.6727 | LR: 3.88e-04 | 2730.8ms/step | 22.0 samples/s
|
| 261 |
+
Step 16350/50462 | Loss: 2.6678 | LR: 3.88e-04 | 2729.5ms/step | 22.0 samples/s
|
| 262 |
+
Step 16360/50462 | Loss: 2.6594 | LR: 3.88e-04 | 2732.6ms/step | 22.0 samples/s
|
| 263 |
+
Step 16370/50462 | Loss: 2.6372 | LR: 3.88e-04 | 2733.8ms/step | 21.9 samples/s
|
| 264 |
+
Step 16380/50462 | Loss: 2.6526 | LR: 3.88e-04 | 2734.1ms/step | 21.9 samples/s
|
| 265 |
+
Step 16390/50462 | Loss: 2.6398 | LR: 3.88e-04 | 2732.3ms/step | 22.0 samples/s
|
| 266 |
+
Step 16400/50462 | Loss: 2.6758 | LR: 3.87e-04 | 2725.7ms/step | 22.0 samples/s
|
| 267 |
+
Step 16410/50462 | Loss: 2.7036 | LR: 3.87e-04 | 2728.4ms/step | 22.0 samples/s
|
| 268 |
+
Step 16420/50462 | Loss: 2.6650 | LR: 3.87e-04 | 2729.6ms/step | 22.0 samples/s
|
| 269 |
+
Step 16430/50462 | Loss: 2.6874 | LR: 3.87e-04 | 2734.4ms/step | 21.9 samples/s
|
| 270 |
+
Step 16440/50462 | Loss: 2.6484 | LR: 3.87e-04 | 2742.3ms/step | 21.9 samples/s
|
| 271 |
+
Step 16450/50462 | Loss: 2.6939 | LR: 3.87e-04 | 2736.2ms/step | 21.9 samples/s
|
| 272 |
+
Step 16460/50462 | Loss: 2.7061 | LR: 3.87e-04 | 2744.9ms/step | 21.9 samples/s
|
| 273 |
+
Step 16470/50462 | Loss: 2.7108 | LR: 3.86e-04 | 2747.9ms/step | 21.8 samples/s
|
| 274 |
+
Step 16480/50462 | Loss: 2.6677 | LR: 3.86e-04 | 2741.1ms/step | 21.9 samples/s
|
| 275 |
+
Step 16490/50462 | Loss: 2.6978 | LR: 3.86e-04 | 2743.5ms/step | 21.9 samples/s
|
| 276 |
+
Step 16500/50462 | Loss: 2.6843 | LR: 3.86e-04 | 2741.2ms/step | 21.9 samples/s
|
| 277 |
+
Validation at step 16500: Loss = 2.6658, Perplexity = 14.38
|
| 278 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-16500
|
| 279 |
+
Step 16510/50462 | Loss: 2.6684 | LR: 3.86e-04 | 4129.2ms/step | 14.5 samples/s
|
| 280 |
+
Step 16520/50462 | Loss: 2.6409 | LR: 3.86e-04 | 2741.7ms/step | 21.9 samples/s
|
| 281 |
+
Step 16530/50462 | Loss: 2.6704 | LR: 3.86e-04 | 2741.7ms/step | 21.9 samples/s
|
| 282 |
+
Step 16540/50462 | Loss: 2.6530 | LR: 3.86e-04 | 2738.3ms/step | 21.9 samples/s
|
| 283 |
+
Step 16550/50462 | Loss: 2.6717 | LR: 3.85e-04 | 2747.7ms/step | 21.8 samples/s
|
| 284 |
+
Step 16560/50462 | Loss: 2.6425 | LR: 3.85e-04 | 2739.8ms/step | 21.9 samples/s
|
| 285 |
+
Step 16570/50462 | Loss: 2.6413 | LR: 3.85e-04 | 2724.9ms/step | 22.0 samples/s
|
| 286 |
+
Step 16580/50462 | Loss: 2.6739 | LR: 3.85e-04 | 2736.8ms/step | 21.9 samples/s
|
epfl_final/train_logs/p2_1p2b_mixed_seed72_4gpu_swapped.log
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
===== SEED72_TO_4GPU_FROM_6500 2026-08-27T12:31:44Z =====
|
| 2 |
+
W0827 12:31:46.763000 3843 site-packages/torch/distributed/run.py:851]
|
| 3 |
+
W0827 12:31:46.763000 3843 site-packages/torch/distributed/run.py:851] *****************************************
|
| 4 |
+
W0827 12:31:46.763000 3843 site-packages/torch/distributed/run.py:851] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 5 |
+
W0827 12:31:46.763000 3843 site-packages/torch/distributed/run.py:851] *****************************************
|
| 6 |
+
Resuming: Using existing output directory /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551
|
| 7 |
+
Running config:
|
| 8 |
+
{
|
| 9 |
+
"model_path": "/mloscratch/homes/navasard/qwen3_06b_base",
|
| 10 |
+
"model_dtype": "bfloat16",
|
| 11 |
+
"init_from_scratch": true,
|
| 12 |
+
"train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
|
| 13 |
+
"val_path": null,
|
| 14 |
+
"max_seq_length": 8192,
|
| 15 |
+
"num_workers": 8,
|
| 16 |
+
"output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551",
|
| 17 |
+
"resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-6500",
|
| 18 |
+
"force_new_run": false,
|
| 19 |
+
"num_epochs": 1,
|
| 20 |
+
"max_steps": 50462,
|
| 21 |
+
"per_device_batch_size": 1,
|
| 22 |
+
"gradient_accumulation_steps": 15,
|
| 23 |
+
"learning_rate": 0.0005,
|
| 24 |
+
"weight_decay": 0.01,
|
| 25 |
+
"warmup_steps": 750,
|
| 26 |
+
"lr_scheduler": "cosine",
|
| 27 |
+
"max_grad_norm": 1.0,
|
| 28 |
+
"save_steps": 250,
|
| 29 |
+
"eval_steps": 250,
|
| 30 |
+
"eval_batches": 50,
|
| 31 |
+
"logging_steps": 10,
|
| 32 |
+
"seed": 72,
|
| 33 |
+
"verbose": true,
|
| 34 |
+
"checkpoint_activations": true,
|
| 35 |
+
"strategy": "ddp",
|
| 36 |
+
"mixed_precision": "bf16",
|
| 37 |
+
"wandb_enabled": true,
|
| 38 |
+
"wandb_project": "stochastic_sparsity",
|
| 39 |
+
"wandb_name": "1.2b mixed 8k seed72 6gpu",
|
| 40 |
+
"wandb_tags": [
|
| 41 |
+
"qwen3",
|
| 42 |
+
"1.2b",
|
| 43 |
+
"8k",
|
| 44 |
+
"p2",
|
| 45 |
+
"mixed",
|
| 46 |
+
"seed72"
|
| 47 |
+
],
|
| 48 |
+
"hidden_size": 1536,
|
| 49 |
+
"intermediate_size": 6144,
|
| 50 |
+
"num_attention_heads": 12,
|
| 51 |
+
"num_key_value_heads": 4,
|
| 52 |
+
"num_hidden_layers": null,
|
| 53 |
+
"head_dim": null,
|
| 54 |
+
"attention_dropout": null,
|
| 55 |
+
"rope_train_position_scale_set": null,
|
| 56 |
+
"attention_type": "sparse_stochastic",
|
| 57 |
+
"sparse_block_size": 128,
|
| 58 |
+
"sparse_num_random_blocks": 35,
|
| 59 |
+
"sparse_mask_pool_size": 10000,
|
| 60 |
+
"sparse_val_dense_compare": false,
|
| 61 |
+
"sparse_layer_indices": null,
|
| 62 |
+
"sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
|
| 63 |
+
"sparse_mask_seed": 72,
|
| 64 |
+
"sparse_mixed_nrb_set": [
|
| 65 |
+
63,
|
| 66 |
+
47,
|
| 67 |
+
31,
|
| 68 |
+
21,
|
| 69 |
+
11
|
| 70 |
+
],
|
| 71 |
+
"dense_mode": false,
|
| 72 |
+
"dense_step_intervals": null
|
| 73 |
+
}
|
| 74 |
+
Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
|
| 75 |
+
Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-6500
|
| 76 |
+
Checkpoint activations: True
|
| 77 |
+
Checkpoint activations: True
|
| 78 |
+
Checkpoint activations: True
|
| 79 |
+
Checkpoint activations: True
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
================================================================================
|
| 83 |
+
MODEL ARCHITECTURE
|
| 84 |
+
================================================================================
|
| 85 |
+
Model config:
|
| 86 |
+
vocab_size: 151936
|
| 87 |
+
hidden_size: 1536
|
| 88 |
+
num_hidden_layers: 28
|
| 89 |
+
num_attention_heads: 12
|
| 90 |
+
num_key_value_heads: 4
|
| 91 |
+
head_dim: 128
|
| 92 |
+
intermediate_size: 6144
|
| 93 |
+
max_position_embeddings: 32768
|
| 94 |
+
rope_theta: 1000000
|
| 95 |
+
rope_train_position_scale_set: None
|
| 96 |
+
attention_type: sparse_stochastic
|
| 97 |
+
sparse_block_size: 128
|
| 98 |
+
sparse_num_random_blocks: 35
|
| 99 |
+
sparse_mask_pool_size: 10000
|
| 100 |
+
sparse_max_seq_length: 8192
|
| 101 |
+
|
| 102 |
+
Parameter count:
|
| 103 |
+
Total parameters: 1,202,352,640
|
| 104 |
+
Trainable parameters: 1,202,352,640
|
| 105 |
+
Non-trainable parameters: 0
|
| 106 |
+
Model size (BF16): 2.40 GB
|
| 107 |
+
|
| 108 |
+
--------------------------------------------------------------------------------
|
| 109 |
+
LAYER-BY-LAYER SUMMARY
|
| 110 |
+
--------------------------------------------------------------------------------
|
| 111 |
+
/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
|
| 112 |
+
|
| 113 |
+
SOLUTION: Use torch.compile(flex_attention)(...)
|
| 114 |
+
|
| 115 |
+
If you want to debug your score_mod/mask_mod, you can set:
|
| 116 |
+
torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
|
| 117 |
+
|
| 118 |
+
This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
|
| 119 |
+
_warn_once(
|
| 120 |
+
================================================================================================================================================================
|
| 121 |
+
Layer (type:depth-idx) Input Shape Output Shape Param # Trainable
|
| 122 |
+
================================================================================================================================================================
|
| 123 |
+
Qwen3ForCausalLM [1, 8192] -- -- True
|
| 124 |
+
├─Qwen3Model: 1-1 -- [1, 8192, 1536] -- True
|
| 125 |
+
│ └─Embedding: 2-1 [1, 8192] [1, 8192, 1536] 233,373,696 True
|
| 126 |
+
│ └─ModuleList: 2-2 -- -- -- True
|
| 127 |
+
│ │ └─Qwen3DecoderLayer: 3-1 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 128 |
+
│ │ └─Qwen3DecoderLayer: 3-2 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 129 |
+
│ │ └─Qwen3DecoderLayer: 3-3 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 130 |
+
│ │ └─Qwen3DecoderLayer: 3-4 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 131 |
+
│ │ └─Qwen3DecoderLayer: 3-5 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 132 |
+
│ │ └─Qwen3DecoderLayer: 3-6 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 133 |
+
│ │ └─Qwen3DecoderLayer: 3-7 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 134 |
+
│ │ └─Qwen3DecoderLayer: 3-8 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 135 |
+
│ │ └─Qwen3DecoderLayer: 3-9 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 136 |
+
│ │ └─Qwen3DecoderLayer: 3-10 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 137 |
+
│ │ └─Qwen3DecoderLayer: 3-11 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 138 |
+
│ │ └─Qwen3DecoderLayer: 3-12 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 139 |
+
│ │ └─Qwen3DecoderLayer: 3-13 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 140 |
+
│ │ └─Qwen3DecoderLayer: 3-14 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 141 |
+
│ │ └─Qwen3DecoderLayer: 3-15 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 142 |
+
│ │ └─Qwen3DecoderLayer: 3-16 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 143 |
+
│ │ └─Qwen3DecoderLayer: 3-17 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 144 |
+
│ │ └─Qwen3DecoderLayer: 3-18 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 145 |
+
│ │ └─Qwen3DecoderLayer: 3-19 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 146 |
+
│ │ └─Qwen3DecoderLayer: 3-20 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 147 |
+
│ │ └─Qwen3DecoderLayer: 3-21 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 148 |
+
│ │ └─Qwen3DecoderLayer: 3-22 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 149 |
+
│ │ └─Qwen3DecoderLayer: 3-23 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 150 |
+
│ │ └─Qwen3DecoderLayer: 3-24 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 151 |
+
│ │ └─Qwen3DecoderLayer: 3-25 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 152 |
+
│ │ └─Qwen3DecoderLayer: 3-26 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 153 |
+
│ │ └─Qwen3DecoderLayer: 3-27 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 154 |
+
│ │ └─Qwen3DecoderLayer: 3-28 [1, 8192, 1536] [1, 8192, 1536] 34,606,336 True
|
| 155 |
+
│ └─Qwen3RMSNorm: 2-3 [1, 8192, 1536] [1, 8192, 1536] 1,536 True
|
| 156 |
+
├─Linear: 1-2 [1, 8192, 1536] [1, 8192, 151936] 233,373,696 True
|
| 157 |
+
================================================================================================================================================================
|
| 158 |
+
Total params: 1,435,726,336
|
| 159 |
+
Trainable params: 1,435,726,336
|
| 160 |
+
Non-trainable params: 0
|
| 161 |
+
Total mult-adds (Units.GIGABYTES): 1.44
|
| 162 |
+
================================================================================================================================================================
|
| 163 |
+
Input size (MB): 0.07
|
| 164 |
+
Forward/backward pass size (MB): 26218.59
|
| 165 |
+
Params size (MB): 2871.45
|
| 166 |
+
Estimated Total Size (MB): 29090.11
|
| 167 |
+
================================================================================================================================================================
|
| 168 |
+
NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M)
|
| 169 |
+
================================================================================
|
| 170 |
+
|
| 171 |
+
Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 172 |
+
Using pre-tokenized dataset (no runtime tokenization)
|
| 173 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 174 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 175 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 176 |
+
Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
|
| 177 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 178 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 179 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 180 |
+
Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
|
| 181 |
+
Dataset split: 3027764 train, 159356 validation
|
| 182 |
+
Training examples (packed, train split): 3027764
|
| 183 |
+
Validation examples (packed, val split): 159356
|
| 184 |
+
Per-device batch size: 1
|
| 185 |
+
World size (GPUs): 4
|
| 186 |
+
Gradient accumulation steps: 15
|
| 187 |
+
Effective batch size (seqs/step): 60
|
| 188 |
+
Sequence length: 8192
|
| 189 |
+
Tokens per optimizer step: 491,520
|
| 190 |
+
Micro-batches per epoch (per rank): 756941
|
| 191 |
+
Optimizer steps per epoch: 50462
|
| 192 |
+
Total optimizer steps: 50462 (capped by max_steps; 1.00 epochs)
|
| 193 |
+
Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-6500...
|
| 194 |
+
Preserving checkpoint LR state for exact resume: current=[0.00048367553572961495], schedule_base=[0.0005]
|
| 195 |
+
Resumed from checkpoint at step 6500 (epoch 0, micro_step 65000)
|
| 196 |
+
Resuming training from step 6500
|
| 197 |
+
Resuming wandb run z1i60278
|
| 198 |
+
wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from WANDB_API_KEY.
|
| 199 |
+
wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
|
| 200 |
+
wandb: setting up run z1i60278
|
| 201 |
+
wandb: Tracking run with wandb version 0.26.1
|
| 202 |
+
wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_123419-z1i60278
|
| 203 |
+
wandb: Run `wandb offline` to turn off syncing.
|
| 204 |
+
wandb: Resuming run 1.2b mixed 8k seed72 6gpu
|
| 205 |
+
wandb: ⭐️ View project at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity
|
| 206 |
+
wandb: 🚀 View run at https://wandb.ai/ruben-navasardyan4869-epfl/stochastic_sparsity/runs/z1i60278
|
| 207 |
+
Starting training...
|
| 208 |
+
Validation at step 6500: Loss = 2.8877, Perplexity = 17.95
|
| 209 |
+
wandb: WARNING Tried to log to step 6500 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 210 |
+
Switching attention mode for step 6501: sparse_stochastic (configured sparse attention)
|
| 211 |
+
Step 6510/50462 | Loss: 2.6826 | LR: 4.84e-04 | 17760.1ms/step | 3.4 samples/s
|
| 212 |
+
wandb: WARNING Tried to log to step 6510 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 213 |
+
Step 6520/50462 | Loss: 2.8271 | LR: 4.84e-04 | 4027.7ms/step | 14.9 samples/s
|
| 214 |
+
wandb: WARNING Tried to log to step 6520 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 215 |
+
Step 6530/50462 | Loss: 2.8107 | LR: 4.84e-04 | 4024.5ms/step | 14.9 samples/s
|
| 216 |
+
wandb: WARNING Tried to log to step 6530 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 217 |
+
Step 6540/50462 | Loss: 2.8026 | LR: 4.83e-04 | 4019.6ms/step | 14.9 samples/s
|
| 218 |
+
wandb: WARNING Tried to log to step 6540 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 219 |
+
Step 6550/50462 | Loss: 2.8097 | LR: 4.83e-04 | 4021.9ms/step | 14.9 samples/s
|
| 220 |
+
wandb: WARNING Tried to log to step 6550 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 221 |
+
Step 6560/50462 | Loss: 2.8077 | LR: 4.83e-04 | 4017.5ms/step | 14.9 samples/s
|
| 222 |
+
wandb: WARNING Tried to log to step 6560 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 223 |
+
Step 6570/50462 | Loss: 2.7977 | LR: 4.83e-04 | 3996.9ms/step | 15.0 samples/s
|
| 224 |
+
wandb: WARNING Tried to log to step 6570 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 225 |
+
Step 6580/50462 | Loss: 2.8219 | LR: 4.83e-04 | 3999.9ms/step | 15.0 samples/s
|
| 226 |
+
wandb: WARNING Tried to log to step 6580 that is less than the current step 6581. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
|
| 227 |
+
Step 6590/50462 | Loss: 2.8186 | LR: 4.83e-04 | 4015.4ms/step | 14.9 samples/s
|
| 228 |
+
Step 6600/50462 | Loss: 2.8482 | LR: 4.83e-04 | 4020.6ms/step | 14.9 samples/s
|
| 229 |
+
Step 6610/50462 | Loss: 2.8191 | LR: 4.83e-04 | 4007.4ms/step | 15.0 samples/s
|
| 230 |
+
Step 6620/50462 | Loss: 2.8197 | LR: 4.83e-04 | 4013.5ms/step | 14.9 samples/s
|
| 231 |
+
Step 6630/50462 | Loss: 2.8011 | LR: 4.83e-04 | 4019.1ms/step | 14.9 samples/s
|
| 232 |
+
Step 6640/50462 | Loss: 2.7943 | LR: 4.83e-04 | 4029.4ms/step | 14.9 samples/s
|
| 233 |
+
Step 6650/50462 | Loss: 2.8284 | LR: 4.83e-04 | 4016.5ms/step | 14.9 samples/s
|
| 234 |
+
Step 6660/50462 | Loss: 2.8044 | LR: 4.83e-04 | 4016.7ms/step | 14.9 samples/s
|
| 235 |
+
Step 6670/50462 | Loss: 2.7611 | LR: 4.83e-04 | 4005.7ms/step | 15.0 samples/s
|
| 236 |
+
Step 6680/50462 | Loss: 2.8229 | LR: 4.83e-04 | 4014.0ms/step | 14.9 samples/s
|
| 237 |
+
Step 6690/50462 | Loss: 2.7947 | LR: 4.83e-04 | 4009.7ms/step | 15.0 samples/s
|
| 238 |
+
Step 6700/50462 | Loss: 2.7909 | LR: 4.83e-04 | 4010.8ms/step | 15.0 samples/s
|
| 239 |
+
Step 6710/50462 | Loss: 2.7850 | LR: 4.82e-04 | 4006.0ms/step | 15.0 samples/s
|
| 240 |
+
Step 6720/50462 | Loss: 2.7956 | LR: 4.82e-04 | 4015.4ms/step | 14.9 samples/s
|
| 241 |
+
Step 6730/50462 | Loss: 2.8085 | LR: 4.82e-04 | 4015.1ms/step | 14.9 samples/s
|
| 242 |
+
Step 6740/50462 | Loss: 2.7803 | LR: 4.82e-04 | 4024.8ms/step | 14.9 samples/s
|
| 243 |
+
Step 6750/50462 | Loss: 2.8210 | LR: 4.82e-04 | 4021.9ms/step | 14.9 samples/s
|
| 244 |
+
Validation at step 6750: Loss = 2.8808, Perplexity = 17.83
|
| 245 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-6750
|
| 246 |
+
Step 6760/50462 | Loss: 2.7952 | LR: 4.82e-04 | 6192.5ms/step | 9.7 samples/s
|
| 247 |
+
Step 6770/50462 | Loss: 2.8053 | LR: 4.82e-04 | 4024.5ms/step | 14.9 samples/s
|
| 248 |
+
Step 6780/50462 | Loss: 2.7691 | LR: 4.82e-04 | 4023.8ms/step | 14.9 samples/s
|
| 249 |
+
Step 6790/50462 | Loss: 2.7909 | LR: 4.82e-04 | 4032.1ms/step | 14.9 samples/s
|
| 250 |
+
Step 6800/50462 | Loss: 2.7701 | LR: 4.82e-04 | 4021.9ms/step | 14.9 samples/s
|
| 251 |
+
Step 6810/50462 | Loss: 2.7817 | LR: 4.82e-04 | 4030.3ms/step | 14.9 samples/s
|
| 252 |
+
Step 6820/50462 | Loss: 2.8258 | LR: 4.82e-04 | 4015.8ms/step | 14.9 samples/s
|
| 253 |
+
Step 6830/50462 | Loss: 2.8073 | LR: 4.82e-04 | 4033.2ms/step | 14.9 samples/s
|
| 254 |
+
Step 6840/50462 | Loss: 2.8070 | LR: 4.82e-04 | 4022.6ms/step | 14.9 samples/s
|
| 255 |
+
Step 6850/50462 | Loss: 2.7906 | LR: 4.82e-04 | 4021.6ms/step | 14.9 samples/s
|
| 256 |
+
Step 6860/50462 | Loss: 2.7759 | LR: 4.82e-04 | 4023.4ms/step | 14.9 samples/s
|
| 257 |
+
Step 6870/50462 | Loss: 2.7745 | LR: 4.82e-04 | 4033.8ms/step | 14.9 samples/s
|
| 258 |
+
Step 6880/50462 | Loss: 2.7598 | LR: 4.81e-04 | 4015.3ms/step | 14.9 samples/s
|
| 259 |
+
Step 6890/50462 | Loss: 2.7938 | LR: 4.81e-04 | 4019.3ms/step | 14.9 samples/s
|
| 260 |
+
Step 6900/50462 | Loss: 2.8027 | LR: 4.81e-04 | 4022.5ms/step | 14.9 samples/s
|
| 261 |
+
Step 6910/50462 | Loss: 2.7734 | LR: 4.81e-04 | 4013.4ms/step | 15.0 samples/s
|
| 262 |
+
Step 6920/50462 | Loss: 2.7688 | LR: 4.81e-04 | 4017.8ms/step | 14.9 samples/s
|
| 263 |
+
Step 6930/50462 | Loss: 2.7731 | LR: 4.81e-04 | 4028.2ms/step | 14.9 samples/s
|
| 264 |
+
Step 6940/50462 | Loss: 2.7985 | LR: 4.81e-04 | 4031.4ms/step | 14.9 samples/s
|
| 265 |
+
Step 6950/50462 | Loss: 2.7878 | LR: 4.81e-04 | 4030.0ms/step | 14.9 samples/s
|
| 266 |
+
Step 6960/50462 | Loss: 2.7882 | LR: 4.81e-04 | 4026.9ms/step | 14.9 samples/s
|
| 267 |
+
Step 6970/50462 | Loss: 2.7773 | LR: 4.81e-04 | 4033.0ms/step | 14.9 samples/s
|
| 268 |
+
Step 6980/50462 | Loss: 2.7751 | LR: 4.81e-04 | 4026.8ms/step | 14.9 samples/s
|
| 269 |
+
Step 6990/50462 | Loss: 2.7785 | LR: 4.81e-04 | 4031.9ms/step | 14.9 samples/s
|
| 270 |
+
Step 7000/50462 | Loss: 2.7497 | LR: 4.81e-04 | 4025.3ms/step | 14.9 samples/s
|
| 271 |
+
Validation at step 7000: Loss = 2.8807, Perplexity = 17.83
|
| 272 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7000
|
| 273 |
+
Step 7010/50462 | Loss: 2.8040 | LR: 4.81e-04 | 5985.9ms/step | 10.0 samples/s
|
| 274 |
+
Step 7020/50462 | Loss: 2.7853 | LR: 4.81e-04 | 4019.1ms/step | 14.9 samples/s
|
| 275 |
+
Step 7030/50462 | Loss: 2.7919 | LR: 4.81e-04 | 4022.9ms/step | 14.9 samples/s
|
| 276 |
+
Step 7040/50462 | Loss: 2.7811 | LR: 4.81e-04 | 4024.4ms/step | 14.9 samples/s
|
| 277 |
+
Step 7050/50462 | Loss: 2.8025 | LR: 4.80e-04 | 4027.3ms/step | 14.9 samples/s
|
| 278 |
+
Step 7060/50462 | Loss: 2.7586 | LR: 4.80e-04 | 4014.0ms/step | 14.9 samples/s
|
| 279 |
+
Step 7070/50462 | Loss: 2.8066 | LR: 4.80e-04 | 4007.4ms/step | 15.0 samples/s
|
| 280 |
+
Step 7080/50462 | Loss: 2.8055 | LR: 4.80e-04 | 4020.3ms/step | 14.9 samples/s
|
| 281 |
+
Step 7090/50462 | Loss: 2.7721 | LR: 4.80e-04 | 4027.5ms/step | 14.9 samples/s
|
| 282 |
+
Step 7100/50462 | Loss: 2.7828 | LR: 4.80e-04 | 4041.1ms/step | 14.8 samples/s
|
| 283 |
+
Step 7110/50462 | Loss: 2.8069 | LR: 4.80e-04 | 4035.9ms/step | 14.9 samples/s
|
| 284 |
+
Step 7120/50462 | Loss: 2.7780 | LR: 4.80e-04 | 4032.4ms/step | 14.9 samples/s
|
| 285 |
+
Step 7130/50462 | Loss: 2.7577 | LR: 4.80e-04 | 4023.5ms/step | 14.9 samples/s
|
| 286 |
+
Step 7140/50462 | Loss: 2.7727 | LR: 4.80e-04 | 4024.4ms/step | 14.9 samples/s
|
| 287 |
+
Step 7150/50462 | Loss: 2.8172 | LR: 4.80e-04 | 4024.1ms/step | 14.9 samples/s
|
| 288 |
+
Step 7160/50462 | Loss: 2.7697 | LR: 4.80e-04 | 4032.5ms/step | 14.9 samples/s
|
| 289 |
+
Step 7170/50462 | Loss: 2.7790 | LR: 4.80e-04 | 4026.4ms/step | 14.9 samples/s
|
| 290 |
+
Step 7180/50462 | Loss: 2.7890 | LR: 4.80e-04 | 4022.0ms/step | 14.9 samples/s
|
| 291 |
+
Step 7190/50462 | Loss: 2.7798 | LR: 4.80e-04 | 4034.1ms/step | 14.9 samples/s
|
| 292 |
+
Step 7200/50462 | Loss: 2.7941 | LR: 4.80e-04 | 4031.8ms/step | 14.9 samples/s
|
| 293 |
+
Step 7210/50462 | Loss: 2.7738 | LR: 4.79e-04 | 4039.8ms/step | 14.9 samples/s
|
| 294 |
+
Step 7220/50462 | Loss: 2.7575 | LR: 4.79e-04 | 4028.4ms/step | 14.9 samples/s
|
| 295 |
+
Step 7230/50462 | Loss: 2.7747 | LR: 4.79e-04 | 4037.9ms/step | 14.9 samples/s
|
| 296 |
+
Step 7240/50462 | Loss: 2.7548 | LR: 4.79e-04 | 4024.0ms/step | 14.9 samples/s
|
| 297 |
+
Step 7250/50462 | Loss: 2.7621 | LR: 4.79e-04 | 4020.7ms/step | 14.9 samples/s
|
| 298 |
+
Validation at step 7250: Loss = 2.8779, Perplexity = 17.78
|
| 299 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7250
|
| 300 |
+
Step 7260/50462 | Loss: 2.7424 | LR: 4.79e-04 | 5997.8ms/step | 10.0 samples/s
|
| 301 |
+
Step 7270/50462 | Loss: 2.7956 | LR: 4.79e-04 | 4022.6ms/step | 14.9 samples/s
|
| 302 |
+
Step 7280/50462 | Loss: 2.8300 | LR: 4.79e-04 | 4018.3ms/step | 14.9 samples/s
|
| 303 |
+
Step 7290/50462 | Loss: 2.7422 | LR: 4.79e-04 | 4021.4ms/step | 14.9 samples/s
|
| 304 |
+
Step 7300/50462 | Loss: 2.7314 | LR: 4.79e-04 | 4017.0ms/step | 14.9 samples/s
|
| 305 |
+
Step 7310/50462 | Loss: 2.7786 | LR: 4.79e-04 | 4024.4ms/step | 14.9 samples/s
|
| 306 |
+
Step 7320/50462 | Loss: 2.7693 | LR: 4.79e-04 | 4015.9ms/step | 14.9 samples/s
|
| 307 |
+
Step 7330/50462 | Loss: 2.8063 | LR: 4.79e-04 | 4024.3ms/step | 14.9 samples/s
|
| 308 |
+
Step 7340/50462 | Loss: 2.7320 | LR: 4.79e-04 | 4019.2ms/step | 14.9 samples/s
|
| 309 |
+
Step 7350/50462 | Loss: 2.8204 | LR: 4.79e-04 | 4020.1ms/step | 14.9 samples/s
|
| 310 |
+
Step 7360/50462 | Loss: 2.7651 | LR: 4.79e-04 | 4024.4ms/step | 14.9 samples/s
|
| 311 |
+
Step 7370/50462 | Loss: 2.7925 | LR: 4.78e-04 | 4021.4ms/step | 14.9 samples/s
|
| 312 |
+
Step 7380/50462 | Loss: 2.7647 | LR: 4.78e-04 | 4019.3ms/step | 14.9 samples/s
|
| 313 |
+
Step 7390/50462 | Loss: 2.7419 | LR: 4.78e-04 | 4020.9ms/step | 14.9 samples/s
|
| 314 |
+
Step 7400/50462 | Loss: 2.7633 | LR: 4.78e-04 | 4012.8ms/step | 15.0 samples/s
|
| 315 |
+
Step 7410/50462 | Loss: 2.7538 | LR: 4.78e-04 | 4020.8ms/step | 14.9 samples/s
|
| 316 |
+
Step 7420/50462 | Loss: 2.7738 | LR: 4.78e-04 | 4029.0ms/step | 14.9 samples/s
|
| 317 |
+
Step 7430/50462 | Loss: 2.7509 | LR: 4.78e-04 | 4024.8ms/step | 14.9 samples/s
|
| 318 |
+
Step 7440/50462 | Loss: 2.7799 | LR: 4.78e-04 | 4006.2ms/step | 15.0 samples/s
|
| 319 |
+
Step 7450/50462 | Loss: 2.7796 | LR: 4.78e-04 | 4014.6ms/step | 14.9 samples/s
|
| 320 |
+
Step 7460/50462 | Loss: 2.7723 | LR: 4.78e-04 | 4007.3ms/step | 15.0 samples/s
|
| 321 |
+
Step 7470/50462 | Loss: 2.7677 | LR: 4.78e-04 | 4020.4ms/step | 14.9 samples/s
|
| 322 |
+
Step 7480/50462 | Loss: 2.8016 | LR: 4.78e-04 | 4015.3ms/step | 14.9 samples/s
|
| 323 |
+
Step 7490/50462 | Loss: 2.8105 | LR: 4.78e-04 | 4014.8ms/step | 14.9 samples/s
|
| 324 |
+
Step 7500/50462 | Loss: 2.7284 | LR: 4.78e-04 | 4017.7ms/step | 14.9 samples/s
|
| 325 |
+
Validation at step 7500: Loss = 2.8755, Perplexity = 17.73
|
| 326 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7500
|
| 327 |
+
Step 7510/50462 | Loss: 2.7743 | LR: 4.78e-04 | 6021.2ms/step | 10.0 samples/s
|
| 328 |
+
Step 7520/50462 | Loss: 2.7873 | LR: 4.77e-04 | 4025.6ms/step | 14.9 samples/s
|
| 329 |
+
Step 7530/50462 | Loss: 2.7742 | LR: 4.77e-04 | 4029.9ms/step | 14.9 samples/s
|
| 330 |
+
Step 7540/50462 | Loss: 2.7739 | LR: 4.77e-04 | 4007.4ms/step | 15.0 samples/s
|
| 331 |
+
Step 7550/50462 | Loss: 2.7705 | LR: 4.77e-04 | 4008.4ms/step | 15.0 samples/s
|
| 332 |
+
Step 7560/50462 | Loss: 2.7870 | LR: 4.77e-04 | 4028.1ms/step | 14.9 samples/s
|
| 333 |
+
Step 7570/50462 | Loss: 2.7973 | LR: 4.77e-04 | 4022.6ms/step | 14.9 samples/s
|
| 334 |
+
Step 7580/50462 | Loss: 2.7616 | LR: 4.77e-04 | 4025.8ms/step | 14.9 samples/s
|
| 335 |
+
Step 7590/50462 | Loss: 2.7551 | LR: 4.77e-04 | 4018.9ms/step | 14.9 samples/s
|
| 336 |
+
Step 7600/50462 | Loss: 2.7627 | LR: 4.77e-04 | 4024.0ms/step | 14.9 samples/s
|
| 337 |
+
Step 7610/50462 | Loss: 2.7718 | LR: 4.77e-04 | 4017.4ms/step | 14.9 samples/s
|
| 338 |
+
Step 7620/50462 | Loss: 2.7159 | LR: 4.77e-04 | 4016.2ms/step | 14.9 samples/s
|
| 339 |
+
Step 7630/50462 | Loss: 2.7783 | LR: 4.77e-04 | 4002.9ms/step | 15.0 samples/s
|
| 340 |
+
Step 7640/50462 | Loss: 2.7606 | LR: 4.77e-04 | 4015.0ms/step | 14.9 samples/s
|
| 341 |
+
Step 7650/50462 | Loss: 2.7420 | LR: 4.77e-04 | 4011.4ms/step | 15.0 samples/s
|
| 342 |
+
Step 7660/50462 | Loss: 2.7552 | LR: 4.77e-04 | 4016.1ms/step | 14.9 samples/s
|
| 343 |
+
Step 7670/50462 | Loss: 2.7698 | LR: 4.76e-04 | 4009.4ms/step | 15.0 samples/s
|
| 344 |
+
Step 7680/50462 | Loss: 2.7787 | LR: 4.76e-04 | 4710.1ms/step | 12.7 samples/s
|
| 345 |
+
Step 7690/50462 | Loss: 2.7543 | LR: 4.76e-04 | 4023.7ms/step | 14.9 samples/s
|
| 346 |
+
Step 7700/50462 | Loss: 2.7901 | LR: 4.76e-04 | 4015.6ms/step | 14.9 samples/s
|
| 347 |
+
Step 7710/50462 | Loss: 2.7695 | LR: 4.76e-04 | 4015.4ms/step | 14.9 samples/s
|
| 348 |
+
Step 7720/50462 | Loss: 2.7441 | LR: 4.76e-04 | 4016.9ms/step | 14.9 samples/s
|
| 349 |
+
Step 7730/50462 | Loss: 2.7593 | LR: 4.76e-04 | 4014.3ms/step | 14.9 samples/s
|
| 350 |
+
Step 7740/50462 | Loss: 2.7722 | LR: 4.76e-04 | 4022.6ms/step | 14.9 samples/s
|
| 351 |
+
Step 7750/50462 | Loss: 2.7420 | LR: 4.76e-04 | 4003.5ms/step | 15.0 samples/s
|
| 352 |
+
Validation at step 7750: Loss = 2.8688, Perplexity = 17.62
|
| 353 |
+
Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-7750
|
| 354 |
+
Step 7760/50462 | Loss: 2.7916 | LR: 4.76e-04 | 5980.0ms/step | 10.0 samples/s
|
| 355 |
+
Step 7770/50462 | Loss: 2.7533 | LR: 4.76e-04 | 4022.4ms/step | 14.9 samples/s
|
| 356 |
+
Step 7780/50462 | Loss: 2.7668 | LR: 4.76e-04 | 4026.3ms/step | 14.9 samples/s
|
| 357 |
+
Step 7790/50462 | Loss: 2.7810 | LR: 4.76e-04 | 4021.4ms/step | 14.9 samples/s
|
| 358 |
+
Step 7800/50462 | Loss: 2.7465 | LR: 4.76e-04 | 4009.1ms/step | 15.0 samples/s
|
| 359 |
+
Step 7810/50462 | Loss: 2.7736 | LR: 4.76e-04 | 4013.9ms/step | 14.9 samples/s
|
| 360 |
+
Step 7820/50462 | Loss: 2.7242 | LR: 4.75e-04 | 4011.9ms/step | 15.0 samples/s
|
| 361 |
+
Step 7830/50462 | Loss: 2.7585 | LR: 4.75e-04 | 4018.6ms/step | 14.9 samples/s
|
| 362 |
+
Step 7840/50462 | Loss: 2.7557 | LR: 4.75e-04 | 4015.0ms/step | 14.9 samples/s
|
epfl_final/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
epfl_final/train_logs/p2_1p2b_mixed_seed72_6gpu_after_dense70.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
7302
|
epfl_final/train_logs/seed70_mixed_48962_battery.log
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[03:51:27] total jobs=22 already-done=0 gpus=[0, 1, 2, 3, 4] n=100
|
| 2 |
+
[03:51:27] START gpu0 passkey_8k_mixed_seed70 try1 (~22 remaining)
|
| 3 |
+
[03:51:27] START gpu1 ppl_8k_mixed_seed70_resampled_fineweb try1 (~22 remaining)
|
| 4 |
+
[03:51:27] START gpu2 ppl_8k_mixed_seed70_resampled_pg19 try1 (~22 remaining)
|
| 5 |
+
[03:51:27] START gpu3 ppl_8k_mixed_seed70_resampled_arxiv try1 (~22 remaining)
|
| 6 |
+
[03:51:27] START gpu4 ppl_8k_mixed_seed70_fixed_fineweb try1 (~22 remaining)
|
| 7 |
+
[03:52:47] DONE gpu0 passkey_8k_mixed_seed70 rc=1 NO-TSV
|
| 8 |
+
[03:52:47] START gpu0 passkey_8k_mixed_seed70 try2 (~22 remaining)
|
| 9 |
+
[03:53:22] DONE gpu0 passkey_8k_mixed_seed70 rc=1 NO-TSV
|
| 10 |
+
[03:53:22] START gpu0 passkey_8k_mixed_seed70 try3 (~21 remaining)
|
| 11 |
+
[03:53:57] DONE gpu0 passkey_8k_mixed_seed70 rc=1 NO-TSV
|
| 12 |
+
[03:53:58] START gpu0 ppl_8k_mixed_seed70_fixed_pg19 try1 (~21 remaining)
|
| 13 |
+
[04:09:28] DONE gpu1 ppl_8k_mixed_seed70_resampled_fineweb rc=0 ok
|
| 14 |
+
[04:09:28] DONE gpu3 ppl_8k_mixed_seed70_resampled_arxiv rc=0 ok
|
| 15 |
+
[04:09:28] DONE gpu4 ppl_8k_mixed_seed70_fixed_fineweb rc=0 ok
|
| 16 |
+
[04:09:28] START gpu1 ppl_8k_mixed_seed70_fixed_arxiv try1 (~18 remaining)
|
| 17 |
+
[04:09:28] START gpu3 ruler_8k_niah_multikey_mixed_seed70 try1 (~18 remaining)
|
| 18 |
+
[04:09:28] START gpu4 ruler_8k_niah_multivalue_mixed_seed70 try1 (~18 remaining)
|
| 19 |
+
[04:09:38] DONE gpu2 ppl_8k_mixed_seed70_resampled_pg19 rc=0 ok
|
| 20 |
+
[04:09:38] START gpu2 ruler_8k_vt_mixed_seed70 try1 (~17 remaining)
|
| 21 |
+
[04:10:08] DONE gpu3 ruler_8k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 22 |
+
[04:10:08] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 23 |
+
[04:10:08] START gpu3 ruler_8k_niah_multikey_mixed_seed70 try2 (~17 remaining)
|
| 24 |
+
[04:10:08] START gpu4 ruler_8k_niah_multivalue_mixed_seed70 try2 (~17 remaining)
|
| 25 |
+
[04:10:13] DONE gpu2 ruler_8k_vt_mixed_seed70 rc=1 NO-TSV
|
| 26 |
+
[04:10:13] START gpu2 ruler_8k_vt_mixed_seed70 try2 (~17 remaining)
|
| 27 |
+
[04:10:43] DONE gpu3 ruler_8k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 28 |
+
[04:10:43] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 29 |
+
[04:10:43] START gpu3 ruler_8k_niah_multikey_mixed_seed70 try3 (~16 remaining)
|
| 30 |
+
[04:10:43] START gpu4 ruler_8k_niah_multivalue_mixed_seed70 try3 (~15 remaining)
|
| 31 |
+
[04:10:48] DONE gpu2 ruler_8k_vt_mixed_seed70 rc=1 NO-TSV
|
| 32 |
+
[04:10:48] START gpu2 ruler_8k_vt_mixed_seed70 try3 (~14 remaining)
|
| 33 |
+
[04:11:23] DONE gpu3 ruler_8k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 34 |
+
[04:11:23] DONE gpu4 ruler_8k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 35 |
+
[04:11:23] DONE gpu2 ruler_8k_vt_mixed_seed70 rc=1 NO-TSV
|
| 36 |
+
[04:11:23] START gpu2 kv_h2o_8k_mixed_seed70 try1 (~14 remaining)
|
| 37 |
+
[04:11:24] START gpu3 kv_stream_8k_mixed_seed70 try1 (~14 remaining)
|
| 38 |
+
[04:11:24] START gpu4 passkey_30k_mixed_seed70 try1 (~14 remaining)
|
| 39 |
+
[04:11:34] DONE gpu0 ppl_8k_mixed_seed70_fixed_pg19 rc=0 ok
|
| 40 |
+
[04:11:34] START gpu0 ppl_30k_mixed_seed70_resampled_pg19 try1 (~13 remaining)
|
| 41 |
+
[04:11:59] DONE gpu4 passkey_30k_mixed_seed70 rc=1 NO-TSV
|
| 42 |
+
[04:11:59] START gpu4 passkey_30k_mixed_seed70 try2 (~13 remaining)
|
| 43 |
+
[04:12:34] DONE gpu4 passkey_30k_mixed_seed70 rc=1 NO-TSV
|
| 44 |
+
[04:12:34] START gpu4 passkey_30k_mixed_seed70 try3 (~12 remaining)
|
| 45 |
+
[04:13:09] DONE gpu4 passkey_30k_mixed_seed70 rc=1 NO-TSV
|
| 46 |
+
[04:13:09] START gpu4 ppl_30k_mixed_seed70_resampled_arxiv try1 (~12 remaining)
|
| 47 |
+
[04:13:54] DONE gpu3 kv_stream_8k_mixed_seed70 rc=0 ok
|
| 48 |
+
[04:13:54] START gpu3 ppl_30k_mixed_seed70_fixed_pg19 try1 (~11 remaining)
|
| 49 |
+
[04:16:29] DONE gpu2 kv_h2o_8k_mixed_seed70 rc=0 ok
|
| 50 |
+
[04:16:29] START gpu2 ppl_30k_mixed_seed70_fixed_arxiv try1 (~10 remaining)
|
| 51 |
+
[04:27:09] DONE gpu1 ppl_8k_mixed_seed70_fixed_arxiv rc=0 ok
|
| 52 |
+
[04:27:09] START gpu1 ruler_30k_niah_multikey_mixed_seed70 try1 (~9 remaining)
|
| 53 |
+
[04:27:49] DONE gpu1 ruler_30k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 54 |
+
[04:27:49] START gpu1 ruler_30k_niah_multikey_mixed_seed70 try2 (~9 remaining)
|
| 55 |
+
[04:28:24] DONE gpu1 ruler_30k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 56 |
+
[04:28:24] START gpu1 ruler_30k_niah_multikey_mixed_seed70 try3 (~8 remaining)
|
| 57 |
+
[04:28:59] DONE gpu1 ruler_30k_niah_multikey_mixed_seed70 rc=1 NO-TSV
|
| 58 |
+
[04:29:00] START gpu1 ruler_30k_niah_multivalue_mixed_seed70 try1 (~8 remaining)
|
| 59 |
+
[04:29:35] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 60 |
+
[04:29:35] START gpu1 ruler_30k_niah_multivalue_mixed_seed70 try2 (~8 remaining)
|
| 61 |
+
[04:30:10] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 62 |
+
[04:30:10] START gpu1 ruler_30k_niah_multivalue_mixed_seed70 try3 (~7 remaining)
|
| 63 |
+
[04:30:45] DONE gpu1 ruler_30k_niah_multivalue_mixed_seed70 rc=1 NO-TSV
|
| 64 |
+
[04:30:45] START gpu1 ruler_30k_vt_mixed_seed70 try1 (~7 remaining)
|
| 65 |
+
[04:31:20] DONE gpu1 ruler_30k_vt_mixed_seed70 rc=1 NO-TSV
|
| 66 |
+
[04:31:20] START gpu1 ruler_30k_vt_mixed_seed70 try2 (~7 remaining)
|
| 67 |
+
[04:31:55] DONE gpu1 ruler_30k_vt_mixed_seed70 rc=1 NO-TSV
|
| 68 |
+
[04:31:55] START gpu1 ruler_30k_vt_mixed_seed70 try3 (~6 remaining)
|
| 69 |
+
[04:32:35] DONE gpu1 ruler_30k_vt_mixed_seed70 rc=1 NO-TSV
|
| 70 |
+
[04:32:35] START gpu1 kv_h2o_30k_mixed_seed70 try1 (~6 remaining)
|
| 71 |
+
[04:37:40] DONE gpu1 kv_h2o_30k_mixed_seed70 rc=0 ok
|
| 72 |
+
[04:37:40] START gpu1 kv_stream_30k_mixed_seed70 try1 (~5 remaining)
|
| 73 |
+
[04:40:15] DONE gpu1 kv_stream_30k_mixed_seed70 rc=0 ok
|
| 74 |
+
[06:45:16] DONE gpu0 ppl_30k_mixed_seed70_resampled_pg19 rc=0 ok
|
| 75 |
+
[06:46:33] DONE gpu4 ppl_30k_mixed_seed70_resampled_arxiv rc=0 ok
|
| 76 |
+
[06:47:24] DONE gpu3 ppl_30k_mixed_seed70_fixed_pg19 rc=0 ok
|
| 77 |
+
[06:49:16] DONE gpu2 ppl_30k_mixed_seed70_fixed_arxiv rc=0 ok
|
| 78 |
+
[06:49:16] === 1.2B BATTERY DONE. done=14/22 failed_or_skipped=['passkey_8k_mixed_seed70', 'ruler_8k_niah_multikey_mixed_seed70', 'ruler_8k_niah_multivalue_mixed_seed70', 'ruler_8k_vt_mixed_seed70', 'passkey_30k_mixed_seed70', 'ruler_30k_niah_multikey_mixed_seed70', 'ruler_30k_niah_multivalue_mixed_seed70', 'ruler_30k_vt_mixed_seed70'] ===
|
epfl_final/train_logs/seed70_mixed_48962_battery_a10080.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[13:12:23] total jobs=22 already-done=14 gpus=[0, 1, 2, 3] n=100
|
| 2 |
+
[13:12:23] START gpu0 passkey_8k_mixed_seed70 try1 (~8 remaining)
|
| 3 |
+
[13:12:23] START gpu1 ruler_8k_niah_multikey_mixed_seed70 try1 (~8 remaining)
|
| 4 |
+
[13:12:23] START gpu2 ruler_8k_niah_multivalue_mixed_seed70 try1 (~8 remaining)
|
| 5 |
+
[13:12:23] START gpu3 ruler_8k_vt_mixed_seed70 try1 (~8 remaining)
|
| 6 |
+
[13:34:18] DONE gpu1 ruler_8k_niah_multikey_mixed_seed70 rc=0 ok
|
| 7 |
+
[13:34:18] START gpu1 passkey_30k_mixed_seed70 try1 (~6 remaining)
|
| 8 |
+
[13:34:23] DONE gpu3 ruler_8k_vt_mixed_seed70 rc=0 ok
|
| 9 |
+
[13:34:23] START gpu3 ruler_30k_niah_multikey_mixed_seed70 try1 (~6 remaining)
|
| 10 |
+
[13:34:33] DONE gpu0 passkey_8k_mixed_seed70 rc=0 ok
|
| 11 |
+
[13:34:33] DONE gpu2 ruler_8k_niah_multivalue_mixed_seed70 rc=0 ok
|
| 12 |
+
[13:34:34] START gpu0 ruler_30k_niah_multivalue_mixed_seed70 try1 (~4 remaining)
|
| 13 |
+
[13:34:34] START gpu2 ruler_30k_vt_mixed_seed70 try1 (~4 remaining)
|