| nohup: ignoring input |
|
|
| ***************************************** |
| Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. |
| ***************************************** |
| New run: Checkpoints will be saved to ./checkpoints/run_20260530_032950 |
| Running config: |
| { |
| "model_path": "/mloscratch/homes/navasard/qwen3_06b_base", |
| "model_dtype": "bfloat16", |
| "init_from_scratch": true, |
| "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized", |
| "val_path": null, |
| "max_seq_length": 8192, |
| "num_workers": 8, |
| "output_dir": "./checkpoints/run_20260530_032950", |
| "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000", |
| "force_new_run": true, |
| "num_epochs": 1, |
| "max_steps": null, |
| "per_device_batch_size": 2, |
| "gradient_accumulation_steps": 10, |
| "learning_rate": 0.0009, |
| "weight_decay": 0.01, |
| "warmup_steps": 750, |
| "lr_scheduler": "cosine", |
| "max_grad_norm": 1.0, |
| "save_steps": 1000, |
| "eval_steps": 250, |
| "eval_batches": 50, |
| "logging_steps": 10, |
| "seed": 42, |
| "verbose": true, |
| "checkpoint_activations": false, |
| "strategy": "ddp", |
| "mixed_precision": "bf16", |
| "wandb_enabled": true, |
| "wandb_project": "stochastic_sparsity", |
| "wandb_name": "dense continued 46k", |
| "wandb_tags": [ |
| "qwen3", |
| "0.6b", |
| "dense-resume" |
| ], |
| "hidden_size": 1536, |
| "intermediate_size": 6144, |
| "num_attention_heads": 12, |
| "num_key_value_heads": 4, |
| "num_hidden_layers": null, |
| "head_dim": null, |
| "attention_type": "sparse_stochastic", |
| "sparse_block_size": 128, |
| "sparse_num_random_blocks": 31, |
| "sparse_mask_pool_size": 10, |
| "sparse_val_dense_compare": false, |
| "sparse_layer_indices": null, |
| "sparse_mask_cache_dir": null, |
| "dense_mode": true |
| } |
| Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base... |
| Resuming from checkpoint: /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000 |
| Checkpoint activations: False |
| Checkpoint activations: False |
| Checkpoint activations: False |
| dense_mode=True: sparse attention layers will use standard causal SDPA. |
|
|
|
|
| ================================================================================ |
| MODEL ARCHITECTURE |
| ================================================================================ |
| Model config: |
| vocab_size: 151936 |
| hidden_size: 1536 |
| num_hidden_layers: 28 |
| num_attention_heads: 12 |
| num_key_value_heads: 4 |
| head_dim: 128 |
| intermediate_size: 6144 |
| max_position_embeddings: 32768 |
| rope_theta: 1000000 |
| attention_type: sparse_stochastic |
| sparse_block_size: 128 |
| sparse_num_random_blocks: 31 |
| sparse_mask_pool_size: 10 |
| sparse_max_seq_length: 8192 |
|
|
| Parameter count: |
| Total parameters: 1,202,352,640 |
| Trainable parameters: 1,202,352,640 |
| Non-trainable parameters: 0 |
| Model size (BF16): 2.40 GB |
|
|
| -------------------------------------------------------------------------------- |
| LAYER-BY-LAYER SUMMARY |
| -------------------------------------------------------------------------------- |
| ================================================================================================================================================================ |
| Layer (type:depth-idx) Input Shape Output Shape Param # Trainable |
| ================================================================================================================================================================ |
| Qwen3ForCausalLM [2, 8192] -- -- True |
| ├─Qwen3Model: 1-1 -- [2, 8192, 1536] -- True |
| │ └─Embedding: 2-1 [2, 8192] [2, 8192, 1536] 233,373,696 True |
| │ └─ModuleList: 2-2 -- -- -- True |
| │ │ └─Qwen3DecoderLayer: 3-1 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-2 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-3 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-4 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-5 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-6 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-7 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-8 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-9 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-10 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-11 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-12 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-13 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-14 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-15 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-16 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-17 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-18 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-19 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-20 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-21 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-22 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-23 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-24 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-25 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-26 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-27 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ │ └─Qwen3DecoderLayer: 3-28 [2, 8192, 1536] [2, 8192, 1536] 34,606,336 True |
| │ └─Qwen3RMSNorm: 2-3 [2, 8192, 1536] [2, 8192, 1536] 1,536 True |
| ├─Linear: 1-2 [2, 8192, 1536] [2, 8192, 151936] 233,373,696 True |
| ================================================================================================================================================================ |
| Total params: 1,435,726,336 |
| Trainable params: 1,435,726,336 |
| Non-trainable params: 0 |
| Total mult-adds (G): 2.87 |
| ================================================================================================================================================================ |
| Input size (MB): 0.13 |
| Forward/backward pass size (MB): 52437.19 |
| Params size (MB): 2871.45 |
| Estimated Total Size (MB): 55308.77 |
| ================================================================================================================================================================ |
| NOTE: torchinfo double-counts tied embed_tokens/lm_head weight (233,373,696 params). True unique parameter count: 1,202,352,640 (1202.4M) |
| ================================================================================ |
|
|
| Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... |
| Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... |
| Using pre-tokenized dataset (no runtime tokenization) |
| Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... |
| Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) |
| Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) |
| Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized... |
| Dataset split: 3027764 train, 159356 validation |
| Training examples (packed, train split): 3027764 |
| Validation examples (packed, val split): 159356 |
| Per-device batch size: 2 |
| World size (GPUs): 3 |
| Gradient accumulation steps: 10 |
| Effective batch size (seqs/step): 60 |
| Sequence length: 8192 |
| Tokens per optimizer step: 491,520 |
| Micro-batches per epoch (per rank): 504627 |
| Optimizer steps per epoch: 50462 |
| Total optimizer steps: 50462 |
| Loading checkpoint from /mloscratch/homes/navasard/sparse/checkpoints/run_20260527_053212/checkpoint-46000... |
| Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120) |
| Overriding checkpoint LR with config LR: 0.0009 |
| Resumed from checkpoint at step 46000 (epoch 0, micro_step 460000) |
| Resuming training from step 46000 |
| wandb: [wandb.login()] Loaded credentials for https: |
| wandb: Currently logged in as: ruben-navasardyan4869 (ruben-navasardyan4869-epfl) to https: |
| wandb: Tracking run with wandb version 0.27.0 |
| wandb: Run data is saved locally in /mloscratch/homes/navasard/sparse/wandb/run-20260530_034853-10196kbl |
| wandb: Run `wandb offline` to turn off syncing. |
| wandb: Syncing run dense continued 46k |
| wandb: ⭐️ View project at https: |
| wandb: 🚀 View run at https: |
| Starting training... |
| Validation at step 46000: Loss = 2.5221, Perplexity = 12.46 |
| Step 46010/50462 | Loss: 2.8364 | LR: 1.77e-05 | 56608.8ms/step | 1.1 samples/s |
| Step 46020/50462 | Loss: 2.6596 | LR: 1.76e-05 | 5187.9ms/step | 11.6 samples/s |
| Step 46030/50462 | Loss: 2.6007 | LR: 1.75e-05 | 5205.0ms/step | 11.5 samples/s |
| Step 46040/50462 | Loss: 2.5616 | LR: 1.75e-05 | 5240.7ms/step | 11.4 samples/s |
| Step 46050/50462 | Loss: 2.5284 | LR: 1.74e-05 | 5239.8ms/step | 11.5 samples/s |
| Step 46060/50462 | Loss: 2.5156 | LR: 1.73e-05 | 5244.0ms/step | 11.4 samples/s |
| Step 46070/50462 | Loss: 2.5310 | LR: 1.72e-05 | 5219.4ms/step | 11.5 samples/s |
| Step 46080/50462 | Loss: 2.5231 | LR: 1.71e-05 | 5235.6ms/step | 11.5 samples/s |
| Step 46090/50462 | Loss: 2.4899 | LR: 1.71e-05 | 5236.2ms/step | 11.5 samples/s |
| Step 46100/50462 | Loss: 2.5151 | LR: 1.70e-05 | 5255.0ms/step | 11.4 samples/s |
| Step 46110/50462 | Loss: 2.5095 | LR: 1.69e-05 | 5254.0ms/step | 11.4 samples/s |
| Step 46120/50462 | Loss: 2.5042 | LR: 1.68e-05 | 5246.2ms/step | 11.4 samples/s |
| Step 46130/50462 | Loss: 2.5049 | LR: 1.68e-05 | 5250.9ms/step | 11.4 samples/s |
| Step 46140/50462 | Loss: 2.5045 | LR: 1.67e-05 | 5239.2ms/step | 11.5 samples/s |
| Step 46150/50462 | Loss: 2.5024 | LR: 1.66e-05 | 5238.1ms/step | 11.5 samples/s |
| Step 46160/50462 | Loss: 2.5194 | LR: 1.65e-05 | 5241.8ms/step | 11.4 samples/s |
| Step 46170/50462 | Loss: 2.5199 | LR: 1.65e-05 | 5240.5ms/step | 11.4 samples/s |
| Step 46180/50462 | Loss: 2.4892 | LR: 1.64e-05 | 5284.6ms/step | 11.4 samples/s |
| Step 46190/50462 | Loss: 2.4895 | LR: 1.63e-05 | 5285.2ms/step | 11.4 samples/s |
| Step 46200/50462 | Loss: 2.4967 | LR: 1.62e-05 | 5258.0ms/step | 11.4 samples/s |
| Step 46210/50462 | Loss: 2.5132 | LR: 1.61e-05 | 5253.4ms/step | 11.4 samples/s |
| Step 46220/50462 | Loss: 2.5161 | LR: 1.61e-05 | 5265.6ms/step | 11.4 samples/s |
| Step 46230/50462 | Loss: 2.5219 | LR: 1.60e-05 | 5297.7ms/step | 11.3 samples/s |
| Step 46240/50462 | Loss: 2.5304 | LR: 1.59e-05 | 5229.3ms/step | 11.5 samples/s |
| Step 46250/50462 | Loss: 2.5282 | LR: 1.58e-05 | 5252.8ms/step | 11.4 samples/s |
| Validation at step 46250: Loss = 2.5195, Perplexity = 12.42 |
| Step 46260/50462 | Loss: 2.5277 | LR: 1.58e-05 | 7824.6ms/step | 7.7 samples/s |
| Step 46270/50462 | Loss: 2.5143 | LR: 1.57e-05 | 5250.4ms/step | 11.4 samples/s |
| Step 46280/50462 | Loss: 2.5458 | LR: 1.56e-05 | 5252.6ms/step | 11.4 samples/s |
| Step 46290/50462 | Loss: 2.5324 | LR: 1.56e-05 | 5248.0ms/step | 11.4 samples/s |
| Step 46300/50462 | Loss: 2.5027 | LR: 1.55e-05 | 5247.2ms/step | 11.4 samples/s |
| Step 46310/50462 | Loss: 2.5101 | LR: 1.54e-05 | 5244.6ms/step | 11.4 samples/s |
| Step 46320/50462 | Loss: 2.5358 | LR: 1.53e-05 | 5243.2ms/step | 11.4 samples/s |
| Step 46330/50462 | Loss: 2.5152 | LR: 1.53e-05 | 5250.3ms/step | 11.4 samples/s |
| Step 46340/50462 | Loss: 2.5148 | LR: 1.52e-05 | 5236.8ms/step | 11.5 samples/s |
| Step 46350/50462 | Loss: 2.5356 | LR: 1.51e-05 | 5245.1ms/step | 11.4 samples/s |
| Step 46360/50462 | Loss: 2.4985 | LR: 1.50e-05 | 5249.4ms/step | 11.4 samples/s |
| Step 46370/50462 | Loss: 2.4840 | LR: 1.50e-05 | 5249.6ms/step | 11.4 samples/s |
| Step 46380/50462 | Loss: 2.5088 | LR: 1.49e-05 | 5245.3ms/step | 11.4 samples/s |
| Step 46390/50462 | Loss: 2.4979 | LR: 1.48e-05 | 5240.0ms/step | 11.5 samples/s |
| Step 46400/50462 | Loss: 2.4894 | LR: 1.47e-05 | 5238.0ms/step | 11.5 samples/s |
| Step 46410/50462 | Loss: 2.5021 | LR: 1.47e-05 | 5247.4ms/step | 11.4 samples/s |
| Step 46420/50462 | Loss: 2.5016 | LR: 1.46e-05 | 5245.3ms/step | 11.4 samples/s |
| Step 46430/50462 | Loss: 2.5224 | LR: 1.45e-05 | 5241.4ms/step | 11.4 samples/s |
| Step 46440/50462 | Loss: 2.5048 | LR: 1.45e-05 | 5243.5ms/step | 11.4 samples/s |
| Step 46450/50462 | Loss: 2.5308 | LR: 1.44e-05 | 5319.2ms/step | 11.3 samples/s |
| Step 46460/50462 | Loss: 2.5089 | LR: 1.43e-05 | 5251.3ms/step | 11.4 samples/s |
| Step 46470/50462 | Loss: 2.5002 | LR: 1.42e-05 | 5250.7ms/step | 11.4 samples/s |
| Step 46480/50462 | Loss: 2.4954 | LR: 1.42e-05 | 5248.2ms/step | 11.4 samples/s |
| Step 46490/50462 | Loss: 2.4914 | LR: 1.41e-05 | 5243.4ms/step | 11.4 samples/s |
| Step 46500/50462 | Loss: 2.5083 | LR: 1.40e-05 | 5276.5ms/step | 11.4 samples/s |
| Validation at step 46500: Loss = 2.5178, Perplexity = 12.40 |
| Step 46510/50462 | Loss: 2.5026 | LR: 1.40e-05 | 7814.7ms/step | 7.7 samples/s |
| Step 46520/50462 | Loss: 2.5260 | LR: 1.39e-05 | 5246.6ms/step | 11.4 samples/s |
| Step 46530/50462 | Loss: 2.5060 | LR: 1.38e-05 | 5254.3ms/step | 11.4 samples/s |
| Step 46540/50462 | Loss: 2.5187 | LR: 1.38e-05 | 5255.4ms/step | 11.4 samples/s |
| Step 46550/50462 | Loss: 2.4805 | LR: 1.37e-05 | 5237.3ms/step | 11.5 samples/s |
| Step 46560/50462 | Loss: 2.4973 | LR: 1.36e-05 | 5247.2ms/step | 11.4 samples/s |
| Step 46570/50462 | Loss: 2.5079 | LR: 1.35e-05 | 5241.1ms/step | 11.4 samples/s |
| Step 46580/50462 | Loss: 2.4997 | LR: 1.35e-05 | 5234.1ms/step | 11.5 samples/s |
| Step 46590/50462 | Loss: 2.4922 | LR: 1.34e-05 | 5245.1ms/step | 11.4 samples/s |
| Step 46600/50462 | Loss: 2.5074 | LR: 1.33e-05 | 5249.5ms/step | 11.4 samples/s |
| Step 46610/50462 | Loss: 2.5054 | LR: 1.33e-05 | 5244.4ms/step | 11.4 samples/s |
| Step 46620/50462 | Loss: 2.4882 | LR: 1.32e-05 | 5242.8ms/step | 11.4 samples/s |
| Step 46630/50462 | Loss: 2.5024 | LR: 1.31e-05 | 5253.4ms/step | 11.4 samples/s |
| Step 46640/50462 | Loss: 2.5145 | LR: 1.31e-05 | 5244.6ms/step | 11.4 samples/s |
| Step 46650/50462 | Loss: 2.5065 | LR: 1.30e-05 | 5250.5ms/step | 11.4 samples/s |
| Step 46660/50462 | Loss: 2.4965 | LR: 1.29e-05 | 5254.2ms/step | 11.4 samples/s |
| Step 46670/50462 | Loss: 2.5113 | LR: 1.29e-05 | 5250.0ms/step | 11.4 samples/s |
| Step 46680/50462 | Loss: 2.5199 | LR: 1.28e-05 | 5246.2ms/step | 11.4 samples/s |
| Step 46690/50462 | Loss: 2.5366 | LR: 1.27e-05 | 5248.3ms/step | 11.4 samples/s |
| Step 46700/50462 | Loss: 2.4968 | LR: 1.27e-05 | 5249.6ms/step | 11.4 samples/s |
| Step 46710/50462 | Loss: 2.5298 | LR: 1.26e-05 | 5273.4ms/step | 11.4 samples/s |
| Step 46720/50462 | Loss: 2.5134 | LR: 1.25e-05 | 5279.3ms/step | 11.4 samples/s |
| Step 46730/50462 | Loss: 2.4932 | LR: 1.25e-05 | 5237.0ms/step | 11.5 samples/s |
| Step 46740/50462 | Loss: 2.5198 | LR: 1.24e-05 | 5227.1ms/step | 11.5 samples/s |
| Step 46750/50462 | Loss: 2.5026 | LR: 1.23e-05 | 5243.4ms/step | 11.4 samples/s |
| Validation at step 46750: Loss = 2.5172, Perplexity = 12.39 |
| Step 46760/50462 | Loss: 2.5171 | LR: 1.23e-05 | 7815.1ms/step | 7.7 samples/s |
| Step 46770/50462 | Loss: 2.5067 | LR: 1.22e-05 | 5292.7ms/step | 11.3 samples/s |
| Step 46780/50462 | Loss: 2.5321 | LR: 1.21e-05 | 5237.2ms/step | 11.5 samples/s |
| Step 46790/50462 | Loss: 2.4894 | LR: 1.21e-05 | 5240.5ms/step | 11.4 samples/s |
| Step 46800/50462 | Loss: 2.5274 | LR: 1.20e-05 | 5245.3ms/step | 11.4 samples/s |
| Step 46810/50462 | Loss: 2.5233 | LR: 1.19e-05 | 5250.8ms/step | 11.4 samples/s |
| Step 46820/50462 | Loss: 2.4867 | LR: 1.19e-05 | 5246.2ms/step | 11.4 samples/s |
| Step 46830/50462 | Loss: 2.4849 | LR: 1.18e-05 | 5246.5ms/step | 11.4 samples/s |
| Step 46840/50462 | Loss: 2.5104 | LR: 1.17e-05 | 5244.4ms/step | 11.4 samples/s |
| Step 46850/50462 | Loss: 2.5064 | LR: 1.17e-05 | 5237.2ms/step | 11.5 samples/s |
| Step 46860/50462 | Loss: 2.5059 | LR: 1.16e-05 | 5243.0ms/step | 11.4 samples/s |
| Step 46870/50462 | Loss: 2.4642 | LR: 1.15e-05 | 5248.1ms/step | 11.4 samples/s |
| Step 46880/50462 | Loss: 2.4996 | LR: 1.15e-05 | 5249.1ms/step | 11.4 samples/s |
| Step 46890/50462 | Loss: 2.5057 | LR: 1.14e-05 | 5249.4ms/step | 11.4 samples/s |
| Step 46900/50462 | Loss: 2.4978 | LR: 1.14e-05 | 5254.3ms/step | 11.4 samples/s |
| Step 46910/50462 | Loss: 2.4909 | LR: 1.13e-05 | 5249.4ms/step | 11.4 samples/s |
| Step 46920/50462 | Loss: 2.5288 | LR: 1.12e-05 | 5249.4ms/step | 11.4 samples/s |
| Step 46930/50462 | Loss: 2.5051 | LR: 1.12e-05 | 5250.7ms/step | 11.4 samples/s |
| Step 46940/50462 | Loss: 2.5071 | LR: 1.11e-05 | 5240.0ms/step | 11.5 samples/s |
| Step 46950/50462 | Loss: 2.4901 | LR: 1.10e-05 | 5248.1ms/step | 11.4 samples/s |
| Step 46960/50462 | Loss: 2.5134 | LR: 1.10e-05 | 5287.1ms/step | 11.3 samples/s |
| Step 46970/50462 | Loss: 2.5171 | LR: 1.09e-05 | 5229.2ms/step | 11.5 samples/s |
| Step 46980/50462 | Loss: 2.5163 | LR: 1.09e-05 | 5229.3ms/step | 11.5 samples/s |
| Step 46990/50462 | Loss: 2.5371 | LR: 1.08e-05 | 5270.8ms/step | 11.4 samples/s |
| Step 47000/50462 | Loss: 2.5167 | LR: 1.07e-05 | 5239.0ms/step | 11.5 samples/s |
| Validation at step 47000: Loss = 2.5168, Perplexity = 12.39 |
| Saved checkpoint to checkpoints/run_20260530_032950/checkpoint-47000 |
| Step 47010/50462 | Loss: 2.5287 | LR: 1.07e-05 | 8566.5ms/step | 7.0 samples/s |
| Step 47020/50462 | Loss: 2.5053 | LR: 1.06e-05 | 5215.9ms/step | 11.5 samples/s |
| Step 47030/50462 | Loss: 2.5252 | LR: 1.05e-05 | 5225.3ms/step | 11.5 samples/s |
| Step 47040/50462 | Loss: 2.4702 | LR: 1.05e-05 | 5266.7ms/step | 11.4 samples/s |
| Step 47050/50462 | Loss: 2.5070 | LR: 1.04e-05 | 5235.6ms/step | 11.5 samples/s |
| Step 47060/50462 | Loss: 2.5053 | LR: 1.04e-05 | 5236.8ms/step | 11.5 samples/s |
| Step 47070/50462 | Loss: 2.4982 | LR: 1.03e-05 | 5239.0ms/step | 11.5 samples/s |
| Step 47080/50462 | Loss: 2.5060 | LR: 1.02e-05 | 5227.9ms/step | 11.5 samples/s |
| Step 47090/50462 | Loss: 2.5009 | LR: 1.02e-05 | 5233.3ms/step | 11.5 samples/s |
| Step 47100/50462 | Loss: 2.5048 | LR: 1.01e-05 | 5237.2ms/step | 11.5 samples/s |
| Step 47110/50462 | Loss: 2.5085 | LR: 1.01e-05 | 5237.8ms/step | 11.5 samples/s |
| Step 47120/50462 | Loss: 2.4590 | LR: 1.00e-05 | 5242.8ms/step | 11.4 samples/s |
| Step 47130/50462 | Loss: 2.5058 | LR: 9.94e-06 | 5245.2ms/step | 11.4 samples/s |
| Step 47140/50462 | Loss: 2.5182 | LR: 9.88e-06 | 5239.9ms/step | 11.5 samples/s |
| Step 47150/50462 | Loss: 2.4744 | LR: 9.82e-06 | 5240.0ms/step | 11.5 samples/s |
| Step 47160/50462 | Loss: 2.5162 | LR: 9.76e-06 | 5237.6ms/step | 11.5 samples/s |
| Step 47170/50462 | Loss: 2.4992 | LR: 9.70e-06 | 5234.2ms/step | 11.5 samples/s |
| Step 47180/50462 | Loss: 2.5086 | LR: 9.64e-06 | 5250.4ms/step | 11.4 samples/s |
| Step 47190/50462 | Loss: 2.5033 | LR: 9.59e-06 | 5250.6ms/step | 11.4 samples/s |
| Step 47200/50462 | Loss: 2.5014 | LR: 9.53e-06 | 5235.9ms/step | 11.5 samples/s |
| Step 47210/50462 | Loss: 2.4751 | LR: 9.47e-06 | 5239.2ms/step | 11.5 samples/s |
| Step 47220/50462 | Loss: 2.5158 | LR: 9.41e-06 | 5266.9ms/step | 11.4 samples/s |
| Step 47230/50462 | Loss: 2.5139 | LR: 9.35e-06 | 5243.6ms/step | 11.4 samples/s |
| Step 47240/50462 | Loss: 2.4825 | LR: 9.30e-06 | 5242.1ms/step | 11.4 samples/s |
| Step 47250/50462 | Loss: 2.5128 | LR: 9.24e-06 | 5241.4ms/step | 11.4 samples/s |
| Validation at step 47250: Loss = 2.5166, Perplexity = 12.39 |
| Step 47260/50462 | Loss: 2.5073 | LR: 9.18e-06 | 7860.4ms/step | 7.6 samples/s |
| Step 47270/50462 | Loss: 2.5191 | LR: 9.12e-06 | 5244.8ms/step | 11.4 samples/s |
| Step 47280/50462 | Loss: 2.4928 | LR: 9.07e-06 | 5242.3ms/step | 11.4 samples/s |
| Step 47290/50462 | Loss: 2.5207 | LR: 9.01e-06 | 5247.2ms/step | 11.4 samples/s |
| Step 47300/50462 | Loss: 2.5191 | LR: 8.95e-06 | 5241.7ms/step | 11.4 samples/s |
| Step 47310/50462 | Loss: 2.4810 | LR: 8.90e-06 | 5283.8ms/step | 11.4 samples/s |
| Step 47320/50462 | Loss: 2.4981 | LR: 8.84e-06 | 5244.8ms/step | 11.4 samples/s |
| Step 47330/50462 | Loss: 2.5013 | LR: 8.79e-06 | 5250.6ms/step | 11.4 samples/s |
| Step 47340/50462 | Loss: 2.4662 | LR: 8.73e-06 | 5257.9ms/step | 11.4 samples/s |
| Step 47350/50462 | Loss: 2.5137 | LR: 8.67e-06 | 5252.8ms/step | 11.4 samples/s |
| Step 47360/50462 | Loss: 2.4842 | LR: 8.62e-06 | 5239.6ms/step | 11.5 samples/s |
| Step 47370/50462 | Loss: 2.5223 | LR: 8.56e-06 | 5234.3ms/step | 11.5 samples/s |
| Step 47380/50462 | Loss: 2.5149 | LR: 8.51e-06 | 5236.9ms/step | 11.5 samples/s |
| Step 47390/50462 | Loss: 2.4977 | LR: 8.45e-06 | 5229.2ms/step | 11.5 samples/s |
| Step 47400/50462 | Loss: 2.5006 | LR: 8.40e-06 | 5240.3ms/step | 11.4 samples/s |
| Step 47410/50462 | Loss: 2.5119 | LR: 8.34e-06 | 5236.1ms/step | 11.5 samples/s |
| Step 47420/50462 | Loss: 2.4639 | LR: 8.29e-06 | 5236.2ms/step | 11.5 samples/s |
| Step 47430/50462 | Loss: 2.4916 | LR: 8.24e-06 | 5240.8ms/step | 11.4 samples/s |
| Step 47440/50462 | Loss: 2.5003 | LR: 8.18e-06 | 5233.6ms/step | 11.5 samples/s |
| Step 47450/50462 | Loss: 2.4752 | LR: 8.13e-06 | 5230.7ms/step | 11.5 samples/s |
| Step 47460/50462 | Loss: 2.5199 | LR: 8.07e-06 | 5228.0ms/step | 11.5 samples/s |
| Step 47470/50462 | Loss: 2.5176 | LR: 8.02e-06 | 5239.7ms/step | 11.5 samples/s |
| Step 47480/50462 | Loss: 2.5164 | LR: 7.97e-06 | 5273.9ms/step | 11.4 samples/s |
| Step 47490/50462 | Loss: 2.5091 | LR: 7.91e-06 | 5210.5ms/step | 11.5 samples/s |
| Step 47500/50462 | Loss: 2.5005 | LR: 7.86e-06 | 5232.8ms/step | 11.5 samples/s |
| Validation at step 47500: Loss = 2.5165, Perplexity = 12.39 |
| Step 47510/50462 | Loss: 2.4714 | LR: 7.81e-06 | 7812.7ms/step | 7.7 samples/s |
| Step 47520/50462 | Loss: 2.5191 | LR: 7.76e-06 | 5273.9ms/step | 11.4 samples/s |
| Step 47530/50462 | Loss: 2.5146 | LR: 7.70e-06 | 5236.1ms/step | 11.5 samples/s |
| Step 47540/50462 | Loss: 2.4830 | LR: 7.65e-06 | 5239.3ms/step | 11.5 samples/s |
| Step 47550/50462 | Loss: 2.5124 | LR: 7.60e-06 | 5247.5ms/step | 11.4 samples/s |
| Step 47560/50462 | Loss: 2.4978 | LR: 7.55e-06 | 5230.9ms/step | 11.5 samples/s |
| Step 47570/50462 | Loss: 2.5079 | LR: 7.49e-06 | 5235.6ms/step | 11.5 samples/s |
| Step 47580/50462 | Loss: 2.5432 | LR: 7.44e-06 | 5269.0ms/step | 11.4 samples/s |
| Step 47590/50462 | Loss: 2.5140 | LR: 7.39e-06 | 5233.3ms/step | 11.5 samples/s |
| Step 47600/50462 | Loss: 2.5048 | LR: 7.34e-06 | 5242.6ms/step | 11.4 samples/s |
| Step 47610/50462 | Loss: 2.4847 | LR: 7.29e-06 | 5245.5ms/step | 11.4 samples/s |
| Step 47620/50462 | Loss: 2.4984 | LR: 7.24e-06 | 5246.2ms/step | 11.4 samples/s |
| Step 47630/50462 | Loss: 2.5256 | LR: 7.19e-06 | 5236.9ms/step | 11.5 samples/s |
| Step 47640/50462 | Loss: 2.4806 | LR: 7.14e-06 | 5242.7ms/step | 11.4 samples/s |
| Step 47650/50462 | Loss: 2.5000 | LR: 7.09e-06 | 5250.1ms/step | 11.4 samples/s |
| Step 47660/50462 | Loss: 2.4990 | LR: 7.04e-06 | 5236.7ms/step | 11.5 samples/s |
| Step 47670/50462 | Loss: 2.4911 | LR: 6.99e-06 | 5233.8ms/step | 11.5 samples/s |
| Step 47680/50462 | Loss: 2.5093 | LR: 6.94e-06 | 5241.7ms/step | 11.4 samples/s |
| Step 47690/50462 | Loss: 2.5050 | LR: 6.89e-06 | 5245.5ms/step | 11.4 samples/s |
| Step 47700/50462 | Loss: 2.5131 | LR: 6.84e-06 | 5251.1ms/step | 11.4 samples/s |
| Step 47710/50462 | Loss: 2.4791 | LR: 6.79e-06 | 5245.5ms/step | 11.4 samples/s |
| 9ms/step | 11.4 samples/s |
| Step 48030/50462 | Loss: 2.4860 | LR: 2.95e-06 | 5289.5ms/step | 11.3 samples/s |
| Step 48040/50462 | Loss: 2.4994 | LR: 2.92e-06 | 5251.1ms/step | 11.4 samples/s |
| Step 48050/50462 | Loss: 2.4927 | LR: 2.90e-06 | 5245.4ms/step | 11.4 samples/s |
| Step 48060/50462 | Loss: 2.5093 | LR: 2.87e-06 | 5250.2ms/step | 11.4 samples/s |
| Step 48070/50462 | Loss: 2.5118 | LR: 2.85e-06 | 5249.8ms/step | 11.4 samples/s |
|
|
