EPFL ALiBi and training provenance 51-100 of 1184

.gitattributes

@@ -203,3 +203,6 @@ epfl_final/SparseAtt/wandb/offline-run-20260921_164533-kbn66izc/run-kbn66izc.wan
 epfl_final/SparseAtt/wandb/offline-run-20260921_164537-zgrsvkbi/run-zgrsvkbi.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/offline-run-20260921_195241-nihe6o4e/run-nihe6o4e.wandb filter=lfs diff=lfs merge=lfs -text
 epfl_final/SparseAtt/wandb/offline-run-20260921_205453-ghfbqjah/run-ghfbqjah.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260430_020702-cxasagq2/run-cxasagq2.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260430_021108-ngpqwy44/run-ngpqwy44.wandb filter=lfs diff=lfs merge=lfs -text
+epfl_final/wandb/run-20260430_021315-cqkg8w61/run-cqkg8w61.wandb filter=lfs diff=lfs merge=lfs -text

epfl_final/configs_dropctl/resume_p28/seed45.json

@@ -0,0 +1,67 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.28
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed45",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 45,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_dropout_p28_seed45",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_dropctl/resume_p28/seed46.json

@@ -0,0 +1,67 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.28
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_dropout_p28_seed46",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 46,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_dropout_p28_seed46",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/calibration/calib_a0p5.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": false,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9948169744088111,
+      0.9846079846079846,
+      0.9710144927536232,
+      0.9365079365079365
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/ropectl_calib_a0p5",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44300,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 0,
+    "lr_scheduler": "constant",
+    "max_grad_norm": 1.0,
+    "save_steps": 300,
+    "eval_steps": 75,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "ropectl_calib_a0p5",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/calibration/calib_a1p0.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": false,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/ropectl_calib_a1p0",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44300,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 0,
+    "lr_scheduler": "constant",
+    "max_grad_norm": 1.0,
+    "save_steps": 300,
+    "eval_steps": 75,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "ropectl_calib_a1p0",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/calibration/calib_a1p5.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": false,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9844509232264333,
+      0.9538239538239538,
+      0.9130434782608695,
+      0.8095238095238093
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/ropectl_calib_a1p5",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44300,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 0,
+    "lr_scheduler": "constant",
+    "max_grad_norm": 1.0,
+    "save_steps": 300,
+    "eval_steps": 75,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "ropectl_calib_a1p5",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/calibration/calib_a2p0.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": false,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9792678976352445,
+      0.9384319384319384,
+      0.8840579710144927,
+      0.7460317460317458
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/ropectl_calib_a2p0",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44300,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 0,
+    "lr_scheduler": "constant",
+    "max_grad_norm": 1.0,
+    "save_steps": 300,
+    "eval_steps": 75,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "ropectl_calib_a2p0",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/calibration/calib_a3p0.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": false,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9689018464528667,
+      0.9076479076479076,
+      0.826086956521739,
+      0.6190476190476187
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": "/mloscratch/homes/navasard/data/fineweb_val_seed42",
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/ropectl_calib_a3p0",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense/run_20260707_034553/checkpoint-44000",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44300,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 5e-05,
+    "weight_decay": 0.01,
+    "warmup_steps": 0,
+    "lr_scheduler": "constant",
+    "max_grad_norm": 1.0,
+    "save_steps": 300,
+    "eval_steps": 75,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "ropectl_calib_a3p0",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/full_a1p0/seed42.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed42",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/full_a1p0/seed43.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 43,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed43",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/full_a1p0/seed44.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 44,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed44",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/full_a1p0/seed45.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 45,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed45",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/full_a1p0/seed46.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46",
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 46,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed46",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_ddp2_a100/seed42.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed42",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control",
+      "ddp2-a100"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_ddp2_a100/seed43.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 43,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed43",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control",
+      "ddp2-a100"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_ddp2_a10040/seed44.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 44,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed44",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control",
+      "ddp2-a100-40gb"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_ddp2_a10040/seed45.json

@@ -0,0 +1,75 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 45,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed45",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control",
+      "ddp2-a100-40gb"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_wandb/seed42.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed42",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 42,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed42",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_wandb/seed43.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed43",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 43,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed43",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_wandb/seed44.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed44",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 44,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed44",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_wandb/seed45.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed45",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 45,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed45",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_ropectl/resume_a1p0_wandb/seed46.json

@@ -0,0 +1,74 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "attention_dropout": 0.0,
+    "rope_train_position_scale_set": [
+      1.0,
+      0.9896339488176222,
+      0.9692159692159692,
+      0.9420289855072463,
+      0.8730158730158729
+    ]
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_ropectl_a1p0_seed46",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 10,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 1000,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 46,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "l8k365m_dense_ropectl_a1p0_seed46",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "rope-control"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed47_ddp2/dense_seed47_ddp2.json

@@ -0,0 +1,70 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47",
+    "resume_from_checkpoint": null,
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 47,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m dense 8k seed47 ddp2",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense",
+      "multiseed",
+      "seed47",
+      "ddp2"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed47_ddp2/mixed_seed47_ddp2.json

@@ -0,0 +1,62 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47",
+    "resume_from_checkpoint": null,
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 47,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m mixed 8k seed47 ddp2",
+    "tags": ["qwen3", "365m", "8k", "mixed", "multiseed", "seed47", "ddp2"]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": false,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 10000,
+    "sparse_mixed_nrb_set": [63, 47, 31, 21, 11],
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed47_dense_ddp5/dense_seed47_ddp5.json

@@ -0,0 +1,70 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed47",
+    "resume_from_checkpoint": null,
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 47,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m dense 8k seed47 ddp5",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense",
+      "multiseed",
+      "seed47",
+      "ddp5"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed47_h100_ddp2/mixed_seed47_h100_ddp2.json

@@ -0,0 +1,62 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed47",
+    "resume_from_checkpoint": null,
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 47,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m mixed 8k seed47 ddp2 h100-resume",
+    "tags": ["qwen3", "365m", "8k", "mixed", "multiseed", "seed47", "ddp2", "h100-resume"]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": false,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 10000,
+    "sparse_mixed_nrb_set": [63, 47, 31, 21, 11],
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed48_dense_ddp5/dense_seed48_ddp5.json

@@ -0,0 +1,70 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48",
+    "resume_from_checkpoint": null,
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 48,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m dense 8k seed48 ddp5",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "scaleladder",
+      "dense",
+      "multiseed",
+      "seed48",
+      "ddp5"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed48_dense_h100_ddp4/dense_seed48_h100_ddp4_resume25000.json

@@ -0,0 +1,71 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48",
+    "resume_from_checkpoint": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_dense_seed48/run_20260821_043824/checkpoint-25000",
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 1,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 48,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m dense 8k seed48 h100 ddp4 resume25k",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "dense",
+      "multiseed",
+      "seed48",
+      "ddp4",
+      "h100",
+      "resume25k"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": true,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 1,
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": null,
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_seed48_mixed_h100_ddp2/mixed_seed48_h100_ddp2.json

@@ -0,0 +1,77 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_seed48",
+    "resume_from_checkpoint": null,
+    "force_new_run": false,
+    "num_epochs": 1,
+    "max_steps": 44000,
+    "per_device_batch_size": 2,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 48,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "stochastic_sparsity",
+    "name": "365m mixed 8k seed48 h100 ddp2",
+    "tags": [
+      "qwen3",
+      "365m",
+      "8k",
+      "mixed",
+      "multiseed",
+      "seed48",
+      "ddp2",
+      "h100"
+    ]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": false,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 10000,
+    "sparse_mixed_nrb_set": [
+      63,
+      47,
+      31,
+      21,
+      11
+    ],
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/configs_speedprobe_seed48_mixed_ddp4/mixed_seed48_ddp4_50step.json

@@ -0,0 +1,62 @@
+{
+  "model": {
+    "path": "/mloscratch/homes/navasard/qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "max_seq_length": 8192,
+    "num_workers": 8
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/l8k365m_mixed_speedprobe_seed48",
+    "resume_from_checkpoint": null,
+    "force_new_run": true,
+    "num_epochs": 1,
+    "max_steps": 50,
+    "per_device_batch_size": 1,
+    "gradient_accumulation_steps": 5,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 1000,
+    "eval_steps": 250,
+    "eval_batches": 50,
+    "logging_steps": 10,
+    "seed": 48,
+    "verbose": true,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "365m mixed 8k speedprobe_seed48 ddp2 h100-resume",
+    "tags": ["qwen3", "365m", "8k", "mixed", "multiseed", "speedprobe_seed48", "ddp2", "h100-resume"]
+  },
+  "attention": {
+    "type": "sparse_stochastic",
+    "dense_mode": false,
+    "sparse_block_size": 128,
+    "sparse_num_random_blocks": 31,
+    "sparse_mask_pool_size": 10000,
+    "sparse_mixed_nrb_set": [63, 47, 31, 21, 11],
+    "sparse_val_dense_compare": false,
+    "dense_step_intervals": null,
+    "sparse_layer_indices": null,
+    "sparse_mask_cache_dir": "/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed",
+    "sparse_mask_seed": 0
+  }
+}

epfl_final/wandb/debug.log

@@ -0,0 +1,19 @@
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_setup.py:_flush():81] Configure stats pid to 508
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260618_175952-33yw8345/logs/debug.log
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260618_175952-33yw8345/logs/debug-internal.log
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:init():766] calling init triggers
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:init():771] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/run_20260618_175928', 'resume_from_checkpoint': None, 'force_new_run': True, 'num_epochs': 1, 'max_steps': None, 'per_device_batch_size': 2, 'gradient_accumulation_steps': 10, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1, 'save_steps': 1000, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': 'sparse 1mask perlayer-fix', 'wandb_tags': ['qwen3', '1.2b', 'sparse_stochastic', '1mask', 'perlayer-fix'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 31, 'sparse_mask_pool_size': 1, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': None, 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:init():814] starting backend
+2026-06-18 17:59:52,448 INFO    MainThread:508 [wandb_init.py:init():829] sending inform_init request
+2026-06-18 17:59:52,770 INFO    MainThread:508 [wandb_init.py:init():834] backend started and connected
+2026-06-18 17:59:52,772 INFO    MainThread:508 [wandb_init.py:init():904] updated telemetry
+2026-06-18 17:59:52,774 INFO    MainThread:508 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout
+2026-06-18 17:59:53,186 INFO    MainThread:508 [wandb_init.py:init():972] starting run threads in backend
+2026-06-18 17:59:53,290 INFO    MainThread:508 [wandb_run.py:_console_start():2523] atexit reg
+2026-06-18 17:59:53,290 INFO    MainThread:508 [wandb_run.py:_redirect():2373] redirect: wrap_raw
+2026-06-18 17:59:53,290 INFO    MainThread:508 [wandb_run.py:_redirect():2442] Wrapping output streams.
+2026-06-18 17:59:53,290 INFO    MainThread:508 [wandb_run.py:_redirect():2465] Redirects installed.
+2026-06-18 17:59:53,293 INFO    MainThread:508 [wandb_init.py:init():1010] run started, returning control to user process

epfl_final/wandb/run-20260430_020702-cxasagq2/files/config.yaml

@@ -0,0 +1,131 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            ocqykme6tsi3am7h67amdeb6fgb1wopc:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "4381286359040"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-d0294574-a180-e984-6ca3-0573e9724f2e
+                host: h100-workspace-4-0-6
+                memory:
+                    total: "1623005847552"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-04-30T02:07:02.327832Z"
+                writerId: ocqykme6tsi3am7h67amdeb6fgb1wopc
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 49
+                - 51
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 15
+                - 16
+                - 61
+            "4": 3.10.12
+            "5": 0.26.1
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: dense
+checkpoint_activations:
+    value: false
+eval_batches:
+    value: 500
+eval_steps:
+    value: 250
+flash_attention:
+    value: true
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 2048
+max_steps:
+    value: 10001
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: ./checkpoints/run_20260430_020658
+per_device_batch_size:
+    value: 8
+resume_from_checkpoint:
+    value: null
+save_steps:
+    value: 2500
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_mask_pool_size:
+    value: 1000
+sparse_num_random_blocks:
+    value: 1
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
+val_path:
+    value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: null
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+warmup_steps:
+    value: 100
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260430_020702-cxasagq2/files/output.log

@@ -0,0 +1,1109 @@
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: dense
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+===========================================================================================================================================================
+Layer (type:depth-idx)                                  Input Shape               Output Shape              Param #                   Trainable
+===========================================================================================================================================================
+Qwen3ForCausalLM                                        [8, 2048]                 --                        --                        True
+├─Qwen3Model: 1-1                                       --                        [8, 2048, 1024]           --                        True
+│    └─Embedding: 2-1                                   [8, 2048]                 [8, 2048, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                  --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                      [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                     [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                [8, 2048, 1024]           [8, 2048, 1024]           1,024                     True
+├─Linear: 1-2                                           [8, 2048, 1024]           [8, 2048, 151936]         155,582,464               True
+===========================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 6.01
+===========================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 42035.31
+Params size (MB): 1503.26
+Estimated Total Size (MB): 43538.71
+===========================================================================================================================================================
+================================================================================
+
+flash_attn is unavailable; skipping flash-attention enablement
+Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
+Loaded 781148 pre-tokenized examples
+Dataset split: 624918 train, 156230 validation
+Training examples: 624918
+Validation examples: 156230
+Steps per epoch: 9764
+Total training steps: 10001
+Starting training...
+Step 10/10001 | Loss: 11.9447 | LR: 5.00e-05 | 7061.7ms/step | 9.1 samples/s
+Step 20/10001 | Loss: 10.6893 | LR: 1.00e-04 | 1851.8ms/step | 34.6 samples/s
+Step 30/10001 | Loss: 9.7331 | LR: 1.50e-04 | 1849.6ms/step | 34.6 samples/s
+Step 40/10001 | Loss: 8.5795 | LR: 2.00e-04 | 1849.0ms/step | 34.6 samples/s
+Step 50/10001 | Loss: 8.0450 | LR: 2.50e-04 | 1850.7ms/step | 34.6 samples/s
+Step 60/10001 | Loss: 7.9444 | LR: 3.00e-04 | 1860.5ms/step | 34.4 samples/s
+Step 70/10001 | Loss: 7.7586 | LR: 3.50e-04 | 1871.2ms/step | 34.2 samples/s
+Step 80/10001 | Loss: 7.5786 | LR: 4.00e-04 | 1874.1ms/step | 34.1 samples/s
+Step 90/10001 | Loss: 7.3969 | LR: 4.50e-04 | 1879.1ms/step | 34.1 samples/s
+Step 100/10001 | Loss: 7.1881 | LR: 5.00e-04 | 1878.9ms/step | 34.1 samples/s
+Step 110/10001 | Loss: 7.0672 | LR: 5.00e-04 | 1878.4ms/step | 34.1 samples/s
+Step 120/10001 | Loss: 6.9168 | LR: 5.00e-04 | 1876.5ms/step | 34.1 samples/s
+Step 130/10001 | Loss: 6.8678 | LR: 5.00e-04 | 1880.1ms/step | 34.0 samples/s
+Step 140/10001 | Loss: 6.7318 | LR: 5.00e-04 | 1880.5ms/step | 34.0 samples/s
+Step 150/10001 | Loss: 6.6755 | LR: 5.00e-04 | 1883.3ms/step | 34.0 samples/s
+Step 160/10001 | Loss: 6.6498 | LR: 5.00e-04 | 1883.8ms/step | 34.0 samples/s
+Step 170/10001 | Loss: 6.5412 | LR: 5.00e-04 | 1881.5ms/step | 34.0 samples/s
+Step 180/10001 | Loss: 6.4740 | LR: 5.00e-04 | 1882.0ms/step | 34.0 samples/s
+Step 190/10001 | Loss: 6.4032 | LR: 5.00e-04 | 1882.0ms/step | 34.0 samples/s
+Step 200/10001 | Loss: 6.3874 | LR: 5.00e-04 | 1881.0ms/step | 34.0 samples/s
+Step 210/10001 | Loss: 6.2891 | LR: 5.00e-04 | 1880.6ms/step | 34.0 samples/s
+Step 220/10001 | Loss: 6.2285 | LR: 5.00e-04 | 1880.1ms/step | 34.0 samples/s
+Step 230/10001 | Loss: 6.1880 | LR: 5.00e-04 | 1879.8ms/step | 34.0 samples/s
+Step 240/10001 | Loss: 6.1449 | LR: 5.00e-04 | 1880.5ms/step | 34.0 samples/s
+Step 250/10001 | Loss: 6.0599 | LR: 5.00e-04 | 1881.3ms/step | 34.0 samples/s
+Validation at step 250: Loss = 6.0395, Perplexity = 419.67
+Step 260/10001 | Loss: 6.0397 | LR: 5.00e-04 | 33955.0ms/step | 1.9 samples/s
+Step 270/10001 | Loss: 5.9703 | LR: 5.00e-04 | 1884.9ms/step | 34.0 samples/s
+Step 280/10001 | Loss: 5.8796 | LR: 5.00e-04 | 1882.1ms/step | 34.0 samples/s
+Step 290/10001 | Loss: 5.8951 | LR: 5.00e-04 | 1881.1ms/step | 34.0 samples/s
+Step 300/10001 | Loss: 5.8447 | LR: 4.99e-04 | 1881.3ms/step | 34.0 samples/s
+Step 310/10001 | Loss: 5.7724 | LR: 4.99e-04 | 1879.0ms/step | 34.1 samples/s
+Step 320/10001 | Loss: 5.7243 | LR: 4.99e-04 | 1883.6ms/step | 34.0 samples/s
+Step 330/10001 | Loss: 5.7488 | LR: 4.99e-04 | 1883.2ms/step | 34.0 samples/s
+Step 340/10001 | Loss: 5.6473 | LR: 4.99e-04 | 1884.8ms/step | 34.0 samples/s
+Step 350/10001 | Loss: 5.6776 | LR: 4.99e-04 | 1885.0ms/step | 34.0 samples/s
+Step 360/10001 | Loss: 5.5965 | LR: 4.99e-04 | 1882.0ms/step | 34.0 samples/s
+Step 370/10001 | Loss: 5.5430 | LR: 4.99e-04 | 1882.6ms/step | 34.0 samples/s
+Step 380/10001 | Loss: 5.5021 | LR: 4.99e-04 | 1883.4ms/step | 34.0 samples/s
+Step 390/10001 | Loss: 5.4627 | LR: 4.99e-04 | 1880.6ms/step | 34.0 samples/s
+Step 400/10001 | Loss: 5.4250 | LR: 4.99e-04 | 1884.0ms/step | 34.0 samples/s
+Step 410/10001 | Loss: 5.3632 | LR: 4.99e-04 | 1884.5ms/step | 34.0 samples/s
+Step 420/10001 | Loss: 5.3085 | LR: 4.99e-04 | 1884.9ms/step | 34.0 samples/s
+Step 430/10001 | Loss: 5.2813 | LR: 4.99e-04 | 1885.9ms/step | 33.9 samples/s
+Step 440/10001 | Loss: 5.2493 | LR: 4.99e-04 | 1882.2ms/step | 34.0 samples/s
+Step 450/10001 | Loss: 5.2082 | LR: 4.98e-04 | 1882.3ms/step | 34.0 samples/s
+Step 460/10001 | Loss: 5.1488 | LR: 4.98e-04 | 1942.0ms/step | 33.0 samples/s
+Step 470/10001 | Loss: 5.1518 | LR: 4.98e-04 | 1887.0ms/step | 33.9 samples/s
+Step 480/10001 | Loss: 5.1005 | LR: 4.98e-04 | 1883.1ms/step | 34.0 samples/s
+Step 490/10001 | Loss: 5.0809 | LR: 4.98e-04 | 1879.4ms/step | 34.1 samples/s
+Step 500/10001 | Loss: 5.0194 | LR: 4.98e-04 | 1883.5ms/step | 34.0 samples/s
+Validation at step 500: Loss = 5.0141, Perplexity = 150.52
+Step 510/10001 | Loss: 4.9929 | LR: 4.98e-04 | 32103.0ms/step | 2.0 samples/s
+Step 520/10001 | Loss: 4.9849 | LR: 4.98e-04 | 1884.1ms/step | 34.0 samples/s
+Step 530/10001 | Loss: 4.9434 | LR: 4.98e-04 | 1883.3ms/step | 34.0 samples/s
+Step 540/10001 | Loss: 4.9031 | LR: 4.98e-04 | 1881.5ms/step | 34.0 samples/s
+Step 550/10001 | Loss: 4.8767 | LR: 4.97e-04 | 1881.5ms/step | 34.0 samples/s
+Step 560/10001 | Loss: 4.8888 | LR: 4.97e-04 | 1883.7ms/step | 34.0 samples/s
+Step 570/10001 | Loss: 4.8422 | LR: 4.97e-04 | 1885.9ms/step | 33.9 samples/s
+Step 580/10001 | Loss: 4.8297 | LR: 4.97e-04 | 1885.9ms/step | 33.9 samples/s
+Step 590/10001 | Loss: 4.7959 | LR: 4.97e-04 | 1882.1ms/step | 34.0 samples/s
+Step 600/10001 | Loss: 4.7920 | LR: 4.97e-04 | 1883.3ms/step | 34.0 samples/s
+Step 610/10001 | Loss: 4.7725 | LR: 4.97e-04 | 1886.2ms/step | 33.9 samples/s
+Step 620/10001 | Loss: 4.7446 | LR: 4.97e-04 | 1884.4ms/step | 34.0 samples/s
+Step 630/10001 | Loss: 4.7199 | LR: 4.96e-04 | 1884.1ms/step | 34.0 samples/s
+Step 640/10001 | Loss: 4.6860 | LR: 4.96e-04 | 1881.8ms/step | 34.0 samples/s
+Step 650/10001 | Loss: 4.6643 | LR: 4.96e-04 | 1881.6ms/step | 34.0 samples/s
+Step 660/10001 | Loss: 4.6818 | LR: 4.96e-04 | 1883.9ms/step | 34.0 samples/s
+Step 670/10001 | Loss: 4.6401 | LR: 4.96e-04 | 1884.0ms/step | 34.0 samples/s
+Step 680/10001 | Loss: 4.6201 | LR: 4.96e-04 | 1884.0ms/step | 34.0 samples/s
+Step 690/10001 | Loss: 4.6105 | LR: 4.96e-04 | 1883.4ms/step | 34.0 samples/s
+Step 700/10001 | Loss: 4.6283 | LR: 4.95e-04 | 1882.6ms/step | 34.0 samples/s
+Step 710/10001 | Loss: 4.5963 | LR: 4.95e-04 | 1883.1ms/step | 34.0 samples/s
+Step 720/10001 | Loss: 4.5989 | LR: 4.95e-04 | 1885.4ms/step | 33.9 samples/s
+Step 730/10001 | Loss: 4.5685 | LR: 4.95e-04 | 1884.9ms/step | 34.0 samples/s
+Step 740/10001 | Loss: 4.5577 | LR: 4.95e-04 | 1884.6ms/step | 34.0 samples/s
+Step 750/10001 | Loss: 4.5409 | LR: 4.95e-04 | 1884.4ms/step | 34.0 samples/s
+Validation at step 750: Loss = 4.5257, Perplexity = 92.36
+Step 760/10001 | Loss: 4.4996 | LR: 4.95e-04 | 32101.4ms/step | 2.0 samples/s
+Step 770/10001 | Loss: 4.5426 | LR: 4.94e-04 | 1883.5ms/step | 34.0 samples/s
+Step 780/10001 | Loss: 4.4758 | LR: 4.94e-04 | 1881.0ms/step | 34.0 samples/s
+Step 790/10001 | Loss: 4.4849 | LR: 4.94e-04 | 1882.1ms/step | 34.0 samples/s
+Step 800/10001 | Loss: 4.4874 | LR: 4.94e-04 | 1882.8ms/step | 34.0 samples/s
+Step 810/10001 | Loss: 4.4591 | LR: 4.94e-04 | 1885.2ms/step | 33.9 samples/s
+Step 820/10001 | Loss: 4.4310 | LR: 4.94e-04 | 1883.9ms/step | 34.0 samples/s
+Step 830/10001 | Loss: 4.4461 | LR: 4.93e-04 | 1883.3ms/step | 34.0 samples/s
+Step 840/10001 | Loss: 4.4067 | LR: 4.93e-04 | 1880.1ms/step | 34.0 samples/s
+Step 850/10001 | Loss: 4.4310 | LR: 4.93e-04 | 1883.4ms/step | 34.0 samples/s
+Step 860/10001 | Loss: 4.3896 | LR: 4.93e-04 | 1884.1ms/step | 34.0 samples/s
+Step 870/10001 | Loss: 4.3729 | LR: 4.93e-04 | 1886.4ms/step | 33.9 samples/s
+Step 880/10001 | Loss: 4.3843 | LR: 4.92e-04 | 1886.9ms/step | 33.9 samples/s
+Step 890/10001 | Loss: 4.3907 | LR: 4.92e-04 | 1884.1ms/step | 34.0 samples/s
+Step 900/10001 | Loss: 4.3472 | LR: 4.92e-04 | 1881.9ms/step | 34.0 samples/s
+Step 910/10001 | Loss: 4.3741 | LR: 4.92e-04 | 1880.6ms/step | 34.0 samples/s
+Step 920/10001 | Loss: 4.3185 | LR: 4.92e-04 | 1882.4ms/step | 34.0 samples/s
+Step 930/10001 | Loss: 4.3262 | LR: 4.91e-04 | 1931.0ms/step | 33.1 samples/s
+Step 940/10001 | Loss: 4.3341 | LR: 4.91e-04 | 1883.2ms/step | 34.0 samples/s
+Step 950/10001 | Loss: 4.2982 | LR: 4.91e-04 | 1882.0ms/step | 34.0 samples/s
+Step 960/10001 | Loss: 4.2670 | LR: 4.91e-04 | 1882.0ms/step | 34.0 samples/s
+Step 970/10001 | Loss: 4.2855 | LR: 4.91e-04 | 1882.3ms/step | 34.0 samples/s
+Step 980/10001 | Loss: 4.2500 | LR: 4.90e-04 | 1886.3ms/step | 33.9 samples/s
+Step 990/10001 | Loss: 4.2781 | LR: 4.90e-04 | 1885.9ms/step | 33.9 samples/s
+Step 1000/10001 | Loss: 4.2812 | LR: 4.90e-04 | 1886.5ms/step | 33.9 samples/s
+Validation at step 1000: Loss = 4.2724, Perplexity = 71.69
+Step 1010/10001 | Loss: 4.2430 | LR: 4.90e-04 | 32087.8ms/step | 2.0 samples/s
+Step 1020/10001 | Loss: 4.2660 | LR: 4.89e-04 | 1884.1ms/step | 34.0 samples/s
+Step 1030/10001 | Loss: 4.2312 | LR: 4.89e-04 | 1883.5ms/step | 34.0 samples/s
+Step 1040/10001 | Loss: 4.2087 | LR: 4.89e-04 | 1883.3ms/step | 34.0 samples/s
+Step 1050/10001 | Loss: 4.2305 | LR: 4.89e-04 | 1882.4ms/step | 34.0 samples/s
+Step 1060/10001 | Loss: 4.2409 | LR: 4.88e-04 | 1884.3ms/step | 34.0 samples/s
+Step 1070/10001 | Loss: 4.2503 | LR: 4.88e-04 | 1884.0ms/step | 34.0 samples/s
+Step 1080/10001 | Loss: 4.1919 | LR: 4.88e-04 | 1880.5ms/step | 34.0 samples/s
+Step 1090/10001 | Loss: 4.1964 | LR: 4.88e-04 | 1881.5ms/step | 34.0 samples/s
+Step 1100/10001 | Loss: 4.1852 | LR: 4.88e-04 | 1882.9ms/step | 34.0 samples/s
+Step 1110/10001 | Loss: 4.1975 | LR: 4.87e-04 | 1883.4ms/step | 34.0 samples/s
+Step 1120/10001 | Loss: 4.1764 | LR: 4.87e-04 | 1883.8ms/step | 34.0 samples/s
+Step 1130/10001 | Loss: 4.1934 | LR: 4.87e-04 | 1882.8ms/step | 34.0 samples/s
+Step 1140/10001 | Loss: 4.1926 | LR: 4.87e-04 | 1883.6ms/step | 34.0 samples/s
+Step 1150/10001 | Loss: 4.1557 | LR: 4.86e-04 | 1885.5ms/step | 33.9 samples/s
+Step 1160/10001 | Loss: 4.1556 | LR: 4.86e-04 | 1884.9ms/step | 34.0 samples/s
+Step 1170/10001 | Loss: 4.1252 | LR: 4.86e-04 | 1881.7ms/step | 34.0 samples/s
+Step 1180/10001 | Loss: 4.1293 | LR: 4.85e-04 | 1880.7ms/step | 34.0 samples/s
+Step 1190/10001 | Loss: 4.1579 | LR: 4.85e-04 | 1882.0ms/step | 34.0 samples/s
+Step 1200/10001 | Loss: 4.1250 | LR: 4.85e-04 | 1883.2ms/step | 34.0 samples/s
+Step 1210/10001 | Loss: 4.1594 | LR: 4.85e-04 | 1883.2ms/step | 34.0 samples/s
+Step 1220/10001 | Loss: 4.1562 | LR: 4.84e-04 | 1879.9ms/step | 34.0 samples/s
+Step 1230/10001 | Loss: 4.1431 | LR: 4.84e-04 | 1880.9ms/step | 34.0 samples/s
+Step 1240/10001 | Loss: 4.1301 | LR: 4.84e-04 | 1881.4ms/step | 34.0 samples/s
+Step 1250/10001 | Loss: 4.1147 | LR: 4.84e-04 | 1883.5ms/step | 34.0 samples/s
+Validation at step 1250: Loss = 4.1004, Perplexity = 60.37
+Step 1260/10001 | Loss: 4.1457 | LR: 4.83e-04 | 32187.0ms/step | 2.0 samples/s
+Step 1270/10001 | Loss: 4.0880 | LR: 4.83e-04 | 1883.4ms/step | 34.0 samples/s
+Step 1280/10001 | Loss: 4.0750 | LR: 4.83e-04 | 1884.9ms/step | 34.0 samples/s
+Step 1290/10001 | Loss: 4.0995 | LR: 4.82e-04 | 1885.2ms/step | 33.9 samples/s
+Step 1300/10001 | Loss: 4.0889 | LR: 4.82e-04 | 1884.6ms/step | 34.0 samples/s
+Step 1310/10001 | Loss: 4.0885 | LR: 4.82e-04 | 1883.9ms/step | 34.0 samples/s
+Step 1320/10001 | Loss: 4.0921 | LR: 4.82e-04 | 1881.4ms/step | 34.0 samples/s
+Step 1330/10001 | Loss: 4.0855 | LR: 4.81e-04 | 1882.2ms/step | 34.0 samples/s
+Step 1340/10001 | Loss: 4.0890 | LR: 4.81e-04 | 1885.7ms/step | 33.9 samples/s
+Step 1350/10001 | Loss: 4.0557 | LR: 4.81e-04 | 1885.2ms/step | 33.9 samples/s
+Step 1360/10001 | Loss: 4.0576 | LR: 4.80e-04 | 1883.0ms/step | 34.0 samples/s
+Step 1370/10001 | Loss: 4.0336 | LR: 4.80e-04 | 1881.4ms/step | 34.0 samples/s
+Step 1380/10001 | Loss: 4.0264 | LR: 4.80e-04 | 1881.1ms/step | 34.0 samples/s
+Step 1390/10001 | Loss: 4.0273 | LR: 4.79e-04 | 1881.5ms/step | 34.0 samples/s
+Step 1400/10001 | Loss: 3.9742 | LR: 4.79e-04 | 1881.3ms/step | 34.0 samples/s
+Step 1410/10001 | Loss: 4.0337 | LR: 4.79e-04 | 1932.2ms/step | 33.1 samples/s
+Step 1420/10001 | Loss: 4.0120 | LR: 4.78e-04 | 1885.2ms/step | 33.9 samples/s
+Step 1430/10001 | Loss: 4.0320 | LR: 4.78e-04 | 1887.4ms/step | 33.9 samples/s
+Step 1440/10001 | Loss: 4.0151 | LR: 4.78e-04 | 1884.3ms/step | 34.0 samples/s
+Step 1450/10001 | Loss: 4.0160 | LR: 4.77e-04 | 1885.5ms/step | 33.9 samples/s
+Step 1460/10001 | Loss: 4.0283 | LR: 4.77e-04 | 1884.0ms/step | 34.0 samples/s
+Step 1470/10001 | Loss: 4.0030 | LR: 4.77e-04 | 1883.9ms/step | 34.0 samples/s
+Step 1480/10001 | Loss: 3.9551 | LR: 4.76e-04 | 1883.0ms/step | 34.0 samples/s
+Step 1490/10001 | Loss: 3.9952 | LR: 4.76e-04 | 1881.0ms/step | 34.0 samples/s
+Step 1500/10001 | Loss: 3.9764 | LR: 4.76e-04 | 1880.9ms/step | 34.0 samples/s
+Validation at step 1500: Loss = 3.9848, Perplexity = 53.78
+Step 1510/10001 | Loss: 3.9917 | LR: 4.75e-04 | 32085.1ms/step | 2.0 samples/s
+Step 1520/10001 | Loss: 3.9653 | LR: 4.75e-04 | 1883.6ms/step | 34.0 samples/s
+Step 1530/10001 | Loss: 3.9785 | LR: 4.75e-04 | 1883.6ms/step | 34.0 samples/s
+Step 1540/10001 | Loss: 3.9644 | LR: 4.74e-04 | 1886.3ms/step | 33.9 samples/s
+Step 1550/10001 | Loss: 3.9439 | LR: 4.74e-04 | 1885.9ms/step | 33.9 samples/s
+Step 1560/10001 | Loss: 3.9685 | LR: 4.74e-04 | 1882.2ms/step | 34.0 samples/s
+Step 1570/10001 | Loss: 3.9553 | LR: 4.73e-04 | 1885.1ms/step | 34.0 samples/s
+Step 1580/10001 | Loss: 3.9378 | LR: 4.73e-04 | 1882.8ms/step | 34.0 samples/s
+Step 1590/10001 | Loss: 3.9612 | LR: 4.73e-04 | 1881.6ms/step | 34.0 samples/s
+Step 1600/10001 | Loss: 3.9324 | LR: 4.72e-04 | 1880.0ms/step | 34.0 samples/s
+Step 1610/10001 | Loss: 3.9226 | LR: 4.72e-04 | 1881.5ms/step | 34.0 samples/s
+Step 1620/10001 | Loss: 3.9384 | LR: 4.71e-04 | 1883.6ms/step | 34.0 samples/s
+Step 1630/10001 | Loss: 3.9221 | LR: 4.71e-04 | 1884.6ms/step | 34.0 samples/s
+Step 1640/10001 | Loss: 3.9102 | LR: 4.71e-04 | 1888.4ms/step | 33.9 samples/s
+Step 1650/10001 | Loss: 3.9148 | LR: 4.70e-04 | 1886.5ms/step | 33.9 samples/s
+Step 1660/10001 | Loss: 3.9369 | LR: 4.70e-04 | 1884.6ms/step | 34.0 samples/s
+Step 1670/10001 | Loss: 3.8977 | LR: 4.70e-04 | 1885.2ms/step | 33.9 samples/s
+Step 1680/10001 | Loss: 3.9225 | LR: 4.69e-04 | 1884.7ms/step | 34.0 samples/s
+Step 1690/10001 | Loss: 3.9033 | LR: 4.69e-04 | 1880.0ms/step | 34.0 samples/s
+Step 1700/10001 | Loss: 3.9082 | LR: 4.68e-04 | 1880.1ms/step | 34.0 samples/s
+Step 1710/10001 | Loss: 3.8929 | LR: 4.68e-04 | 1884.5ms/step | 34.0 samples/s
+Step 1720/10001 | Loss: 3.8859 | LR: 4.68e-04 | 1884.0ms/step | 34.0 samples/s
+Step 1730/10001 | Loss: 3.9195 | LR: 4.67e-04 | 1886.3ms/step | 33.9 samples/s
+Step 1740/10001 | Loss: 3.8911 | LR: 4.67e-04 | 1883.3ms/step | 34.0 samples/s
+Step 1750/10001 | Loss: 3.9195 | LR: 4.67e-04 | 1880.5ms/step | 34.0 samples/s
+Validation at step 1750: Loss = 3.8961, Perplexity = 49.21
+Step 1760/10001 | Loss: 3.9077 | LR: 4.66e-04 | 32085.9ms/step | 2.0 samples/s
+Step 1770/10001 | Loss: 3.8634 | LR: 4.66e-04 | 1885.4ms/step | 33.9 samples/s
+Step 1780/10001 | Loss: 3.8772 | LR: 4.65e-04 | 1880.7ms/step | 34.0 samples/s
+Step 1790/10001 | Loss: 3.8907 | LR: 4.65e-04 | 1878.6ms/step | 34.1 samples/s
+Step 1800/10001 | Loss: 3.8815 | LR: 4.65e-04 | 1881.1ms/step | 34.0 samples/s
+Step 1810/10001 | Loss: 3.8686 | LR: 4.64e-04 | 1883.0ms/step | 34.0 samples/s
+Step 1820/10001 | Loss: 3.8572 | LR: 4.64e-04 | 1879.6ms/step | 34.1 samples/s
+Step 1830/10001 | Loss: 3.8806 | LR: 4.63e-04 | 1880.5ms/step | 34.0 samples/s
+Step 1840/10001 | Loss: 3.8457 | LR: 4.63e-04 | 1882.5ms/step | 34.0 samples/s
+Step 1850/10001 | Loss: 3.8551 | LR: 4.62e-04 | 1883.0ms/step | 34.0 samples/s
+Step 1860/10001 | Loss: 3.8582 | LR: 4.62e-04 | 1880.0ms/step | 34.0 samples/s
+Step 1870/10001 | Loss: 3.8534 | LR: 4.62e-04 | 1880.9ms/step | 34.0 samples/s
+Step 1880/10001 | Loss: 3.8551 | LR: 4.61e-04 | 1931.1ms/step | 33.1 samples/s
+Step 1890/10001 | Loss: 3.8417 | LR: 4.61e-04 | 1879.5ms/step | 34.1 samples/s
+Step 1900/10001 | Loss: 3.8629 | LR: 4.60e-04 | 1881.5ms/step | 34.0 samples/s
+Step 1910/10001 | Loss: 3.8549 | LR: 4.60e-04 | 1879.5ms/step | 34.1 samples/s
+Step 1920/10001 | Loss: 3.8627 | LR: 4.59e-04 | 1881.0ms/step | 34.0 samples/s
+Step 1930/10001 | Loss: 3.8506 | LR: 4.59e-04 | 1885.6ms/step | 33.9 samples/s
+Step 1940/10001 | Loss: 3.8302 | LR: 4.59e-04 | 1883.5ms/step | 34.0 samples/s
+Step 1950/10001 | Loss: 3.8411 | LR: 4.58e-04 | 1883.9ms/step | 34.0 samples/s
+Step 1960/10001 | Loss: 3.8413 | LR: 4.58e-04 | 1880.3ms/step | 34.0 samples/s
+Step 1970/10001 | Loss: 3.8178 | LR: 4.57e-04 | 1879.6ms/step | 34.0 samples/s
+Step 1980/10001 | Loss: 3.8370 | LR: 4.57e-04 | 1880.7ms/step | 34.0 samples/s
+Step 1990/10001 | Loss: 3.8327 | LR: 4.56e-04 | 1880.9ms/step | 34.0 samples/s
+Step 2000/10001 | Loss: 3.8115 | LR: 4.56e-04 | 1880.5ms/step | 34.0 samples/s
+Validation at step 2000: Loss = 3.8235, Perplexity = 45.76
+Step 2010/10001 | Loss: 3.8183 | LR: 4.55e-04 | 32052.3ms/step | 2.0 samples/s
+Step 2020/10001 | Loss: 3.8416 | LR: 4.55e-04 | 1882.3ms/step | 34.0 samples/s
+Step 2030/10001 | Loss: 3.8068 | LR: 4.55e-04 | 1884.0ms/step | 34.0 samples/s
+Step 2040/10001 | Loss: 3.7921 | LR: 4.54e-04 | 1884.0ms/step | 34.0 samples/s
+Step 2050/10001 | Loss: 3.8149 | LR: 4.54e-04 | 1878.9ms/step | 34.1 samples/s
+Step 2060/10001 | Loss: 3.8119 | LR: 4.53e-04 | 1882.6ms/step | 34.0 samples/s
+Step 2070/10001 | Loss: 3.7922 | LR: 4.53e-04 | 1883.4ms/step | 34.0 samples/s
+Step 2080/10001 | Loss: 3.8415 | LR: 4.52e-04 | 1880.6ms/step | 34.0 samples/s
+Step 2090/10001 | Loss: 3.7916 | LR: 4.52e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2100/10001 | Loss: 3.7933 | LR: 4.51e-04 | 1881.4ms/step | 34.0 samples/s
+Step 2110/10001 | Loss: 3.7739 | LR: 4.51e-04 | 1881.9ms/step | 34.0 samples/s
+Step 2120/10001 | Loss: 3.8026 | LR: 4.50e-04 | 1879.5ms/step | 34.1 samples/s
+Step 2130/10001 | Loss: 3.7752 | LR: 4.50e-04 | 1879.1ms/step | 34.1 samples/s
+Step 2140/10001 | Loss: 3.7984 | LR: 4.49e-04 | 1882.0ms/step | 34.0 samples/s
+Step 2150/10001 | Loss: 3.7790 | LR: 4.49e-04 | 1879.5ms/step | 34.1 samples/s
+Step 2160/10001 | Loss: 3.7722 | LR: 4.48e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2170/10001 | Loss: 3.7930 | LR: 4.48e-04 | 1883.2ms/step | 34.0 samples/s
+Step 2180/10001 | Loss: 3.7903 | LR: 4.48e-04 | 1879.5ms/step | 34.1 samples/s
+Step 2190/10001 | Loss: 3.7858 | LR: 4.47e-04 | 1878.6ms/step | 34.1 samples/s
+Step 2200/10001 | Loss: 3.7899 | LR: 4.47e-04 | 1880.1ms/step | 34.0 samples/s
+Step 2210/10001 | Loss: 3.7421 | LR: 4.46e-04 | 1879.9ms/step | 34.0 samples/s
+Step 2220/10001 | Loss: 3.7852 | LR: 4.46e-04 | 1882.1ms/step | 34.0 samples/s
+Step 2230/10001 | Loss: 3.7866 | LR: 4.45e-04 | 1882.0ms/step | 34.0 samples/s
+Step 2240/10001 | Loss: 3.7584 | LR: 4.45e-04 | 1882.7ms/step | 34.0 samples/s
+Step 2250/10001 | Loss: 3.7229 | LR: 4.44e-04 | 1881.9ms/step | 34.0 samples/s
+Validation at step 2250: Loss = 3.7632, Perplexity = 43.08
+Step 2260/10001 | Loss: 3.7658 | LR: 4.44e-04 | 32045.2ms/step | 2.0 samples/s
+Step 2270/10001 | Loss: 3.7800 | LR: 4.43e-04 | 1883.7ms/step | 34.0 samples/s
+Step 2280/10001 | Loss: 3.7526 | LR: 4.43e-04 | 1882.2ms/step | 34.0 samples/s
+Step 2290/10001 | Loss: 3.7820 | LR: 4.42e-04 | 1880.3ms/step | 34.0 samples/s
+Step 2300/10001 | Loss: 3.7920 | LR: 4.42e-04 | 1882.5ms/step | 34.0 samples/s
+Step 2310/10001 | Loss: 3.7736 | LR: 4.41e-04 | 1883.5ms/step | 34.0 samples/s
+Step 2320/10001 | Loss: 3.7427 | LR: 4.40e-04 | 1883.0ms/step | 34.0 samples/s
+Step 2330/10001 | Loss: 3.7333 | LR: 4.40e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2340/10001 | Loss: 3.7487 | LR: 4.39e-04 | 1880.4ms/step | 34.0 samples/s
+Step 2350/10001 | Loss: 3.7734 | LR: 4.39e-04 | 1881.3ms/step | 34.0 samples/s
+Step 2360/10001 | Loss: 3.7509 | LR: 4.38e-04 | 1930.8ms/step | 33.1 samples/s
+Step 2370/10001 | Loss: 3.7574 | LR: 4.38e-04 | 1883.2ms/step | 34.0 samples/s
+Step 2380/10001 | Loss: 3.7414 | LR: 4.37e-04 | 1883.5ms/step | 34.0 samples/s
+Step 2390/10001 | Loss: 3.7173 | LR: 4.37e-04 | 1884.9ms/step | 34.0 samples/s
+Step 2400/10001 | Loss: 3.7314 | LR: 4.36e-04 | 1885.3ms/step | 33.9 samples/s
+Step 2410/10001 | Loss: 3.7552 | LR: 4.36e-04 | 1885.8ms/step | 33.9 samples/s
+Step 2420/10001 | Loss: 3.7286 | LR: 4.35e-04 | 1879.3ms/step | 34.1 samples/s
+Step 2430/10001 | Loss: 3.7204 | LR: 4.35e-04 | 1880.1ms/step | 34.0 samples/s
+Step 2440/10001 | Loss: 3.7152 | LR: 4.34e-04 | 1883.1ms/step | 34.0 samples/s
+Step 2450/10001 | Loss: 3.7247 | LR: 4.34e-04 | 1880.4ms/step | 34.0 samples/s
+Step 2460/10001 | Loss: 3.7103 | LR: 4.33e-04 | 1879.9ms/step | 34.0 samples/s
+Step 2470/10001 | Loss: 3.7392 | LR: 4.33e-04 | 1884.5ms/step | 34.0 samples/s
+Step 2480/10001 | Loss: 3.7361 | LR: 4.32e-04 | 1884.0ms/step | 34.0 samples/s
+Step 2490/10001 | Loss: 3.7187 | LR: 4.31e-04 | 1884.7ms/step | 34.0 samples/s
+Step 2500/10001 | Loss: 3.7217 | LR: 4.31e-04 | 1883.8ms/step | 34.0 samples/s
+Validation at step 2500: Loss = 3.7157, Perplexity = 41.09
+Saved checkpoint to checkpoints/run_20260430_020658/checkpoint-2500
+Step 2510/10001 | Loss: 3.7049 | LR: 4.30e-04 | 32578.9ms/step | 2.0 samples/s
+Step 2520/10001 | Loss: 3.6830 | LR: 4.30e-04 | 1881.0ms/step | 34.0 samples/s
+Step 2530/10001 | Loss: 3.7065 | LR: 4.29e-04 | 1880.9ms/step | 34.0 samples/s
+Step 2540/10001 | Loss: 3.7092 | LR: 4.29e-04 | 1881.6ms/step | 34.0 samples/s
+Step 2550/10001 | Loss: 3.7248 | LR: 4.28e-04 | 1880.6ms/step | 34.0 samples/s
+Step 2560/10001 | Loss: 3.6633 | LR: 4.28e-04 | 1881.8ms/step | 34.0 samples/s
+Step 2570/10001 | Loss: 3.7024 | LR: 4.27e-04 | 1881.9ms/step | 34.0 samples/s
+Step 2580/10001 | Loss: 3.6926 | LR: 4.27e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2590/10001 | Loss: 3.7063 | LR: 4.26e-04 | 1880.6ms/step | 34.0 samples/s
+Step 2600/10001 | Loss: 3.6844 | LR: 4.25e-04 | 1881.4ms/step | 34.0 samples/s
+Step 2610/10001 | Loss: 3.7291 | LR: 4.25e-04 | 1881.0ms/step | 34.0 samples/s
+Step 2620/10001 | Loss: 3.6826 | LR: 4.24e-04 | 1880.8ms/step | 34.0 samples/s
+Step 2630/10001 | Loss: 3.6769 | LR: 4.24e-04 | 1880.1ms/step | 34.0 samples/s
+Step 2640/10001 | Loss: 3.6811 | LR: 4.23e-04 | 1880.6ms/step | 34.0 samples/s
+Step 2650/10001 | Loss: 3.6574 | LR: 4.23e-04 | 1882.4ms/step | 34.0 samples/s
+Step 2660/10001 | Loss: 3.6927 | LR: 4.22e-04 | 1882.5ms/step | 34.0 samples/s
+Step 2670/10001 | Loss: 3.7164 | LR: 4.21e-04 | 1882.0ms/step | 34.0 samples/s
+Step 2680/10001 | Loss: 3.6792 | LR: 4.21e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2690/10001 | Loss: 3.6567 | LR: 4.20e-04 | 1879.4ms/step | 34.1 samples/s
+Step 2700/10001 | Loss: 3.6768 | LR: 4.20e-04 | 1880.0ms/step | 34.0 samples/s
+Step 2710/10001 | Loss: 3.6709 | LR: 4.19e-04 | 1882.5ms/step | 34.0 samples/s
+Step 2720/10001 | Loss: 3.6498 | LR: 4.18e-04 | 1883.2ms/step | 34.0 samples/s
+Step 2730/10001 | Loss: 3.6648 | LR: 4.18e-04 | 1883.8ms/step | 34.0 samples/s
+Step 2740/10001 | Loss: 3.6781 | LR: 4.17e-04 | 1883.6ms/step | 34.0 samples/s
+Step 2750/10001 | Loss: 3.6917 | LR: 4.17e-04 | 1880.9ms/step | 34.0 samples/s
+Validation at step 2750: Loss = 3.6721, Perplexity = 39.34
+Step 2760/10001 | Loss: 3.6801 | LR: 4.16e-04 | 32065.0ms/step | 2.0 samples/s
+Step 2770/10001 | Loss: 3.6918 | LR: 4.16e-04 | 1881.4ms/step | 34.0 samples/s
+Step 2780/10001 | Loss: 3.6577 | LR: 4.15e-04 | 1881.5ms/step | 34.0 samples/s
+Step 2790/10001 | Loss: 3.6920 | LR: 4.14e-04 | 1882.9ms/step | 34.0 samples/s
+Step 2800/10001 | Loss: 3.6675 | LR: 4.14e-04 | 1883.0ms/step | 34.0 samples/s
+Step 2810/10001 | Loss: 3.6417 | LR: 4.13e-04 | 1880.1ms/step | 34.0 samples/s
+Step 2820/10001 | Loss: 3.6668 | LR: 4.13e-04 | 1882.9ms/step | 34.0 samples/s
+Step 2830/10001 | Loss: 3.6743 | LR: 4.12e-04 | 1928.9ms/step | 33.2 samples/s
+Step 2840/10001 | Loss: 3.6670 | LR: 4.11e-04 | 1881.1ms/step | 34.0 samples/s
+Step 2850/10001 | Loss: 3.6568 | LR: 4.11e-04 | 1879.6ms/step | 34.1 samples/s
+Step 2860/10001 | Loss: 3.6595 | LR: 4.10e-04 | 1883.4ms/step | 34.0 samples/s
+Step 2870/10001 | Loss: 3.6793 | LR: 4.09e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2880/10001 | Loss: 3.6820 | LR: 4.09e-04 | 1880.6ms/step | 34.0 samples/s
+Step 2890/10001 | Loss: 3.6820 | LR: 4.08e-04 | 1880.0ms/step | 34.0 samples/s
+Step 2900/10001 | Loss: 3.6388 | LR: 4.08e-04 | 1883.5ms/step | 34.0 samples/s
+Step 2910/10001 | Loss: 3.6397 | LR: 4.07e-04 | 1879.9ms/step | 34.0 samples/s
+Step 2920/10001 | Loss: 3.6523 | LR: 4.06e-04 | 1881.6ms/step | 34.0 samples/s
+Step 2930/10001 | Loss: 3.6619 | LR: 4.06e-04 | 1883.8ms/step | 34.0 samples/s
+Step 2940/10001 | Loss: 3.6155 | LR: 4.05e-04 | 1883.3ms/step | 34.0 samples/s
+Step 2950/10001 | Loss: 3.6224 | LR: 4.05e-04 | 1883.9ms/step | 34.0 samples/s
+Step 2960/10001 | Loss: 3.6352 | LR: 4.04e-04 | 1881.4ms/step | 34.0 samples/s
+Step 2970/10001 | Loss: 3.6398 | LR: 4.03e-04 | 1881.3ms/step | 34.0 samples/s
+Step 2980/10001 | Loss: 3.6404 | LR: 4.03e-04 | 1880.5ms/step | 34.0 samples/s
+Step 2990/10001 | Loss: 3.6520 | LR: 4.02e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3000/10001 | Loss: 3.6242 | LR: 4.01e-04 | 1882.8ms/step | 34.0 samples/s
+Validation at step 3000: Loss = 3.6351, Perplexity = 37.91
+Step 3010/10001 | Loss: 3.6712 | LR: 4.01e-04 | 32091.3ms/step | 2.0 samples/s
+Step 3020/10001 | Loss: 3.6450 | LR: 4.00e-04 | 1880.6ms/step | 34.0 samples/s
+Step 3030/10001 | Loss: 3.6284 | LR: 4.00e-04 | 1886.3ms/step | 33.9 samples/s
+Step 3040/10001 | Loss: 3.6122 | LR: 3.99e-04 | 1883.4ms/step | 34.0 samples/s
+Step 3050/10001 | Loss: 3.6273 | LR: 3.98e-04 | 1883.7ms/step | 34.0 samples/s
+Step 3060/10001 | Loss: 3.6346 | LR: 3.98e-04 | 1883.5ms/step | 34.0 samples/s
+Step 3070/10001 | Loss: 3.5939 | LR: 3.97e-04 | 1883.1ms/step | 34.0 samples/s
+Step 3080/10001 | Loss: 3.6033 | LR: 3.96e-04 | 1883.3ms/step | 34.0 samples/s
+Step 3090/10001 | Loss: 3.6301 | LR: 3.96e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3100/10001 | Loss: 3.6017 | LR: 3.95e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3110/10001 | Loss: 3.6515 | LR: 3.94e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3120/10001 | Loss: 3.6374 | LR: 3.94e-04 | 1880.4ms/step | 34.0 samples/s
+Step 3130/10001 | Loss: 3.6103 | LR: 3.93e-04 | 1881.2ms/step | 34.0 samples/s
+Step 3140/10001 | Loss: 3.6222 | LR: 3.92e-04 | 1880.4ms/step | 34.0 samples/s
+Step 3150/10001 | Loss: 3.6171 | LR: 3.92e-04 | 1880.5ms/step | 34.0 samples/s
+Step 3160/10001 | Loss: 3.5851 | LR: 3.91e-04 | 1882.1ms/step | 34.0 samples/s
+Step 3170/10001 | Loss: 3.5892 | LR: 3.90e-04 | 1883.4ms/step | 34.0 samples/s
+Step 3180/10001 | Loss: 3.6128 | LR: 3.90e-04 | 1883.0ms/step | 34.0 samples/s
+Step 3190/10001 | Loss: 3.6304 | LR: 3.89e-04 | 1883.4ms/step | 34.0 samples/s
+Step 3200/10001 | Loss: 3.5975 | LR: 3.89e-04 | 1883.8ms/step | 34.0 samples/s
+Step 3210/10001 | Loss: 3.5917 | LR: 3.88e-04 | 1881.7ms/step | 34.0 samples/s
+Step 3220/10001 | Loss: 3.6145 | LR: 3.87e-04 | 1882.5ms/step | 34.0 samples/s
+Step 3230/10001 | Loss: 3.6000 | LR: 3.87e-04 | 1882.2ms/step | 34.0 samples/s
+Step 3240/10001 | Loss: 3.5865 | LR: 3.86e-04 | 1880.7ms/step | 34.0 samples/s
+Step 3250/10001 | Loss: 3.5935 | LR: 3.85e-04 | 1880.0ms/step | 34.0 samples/s
+Validation at step 3250: Loss = 3.6019, Perplexity = 36.67
+Step 3260/10001 | Loss: 3.6106 | LR: 3.85e-04 | 32069.6ms/step | 2.0 samples/s
+Step 3270/10001 | Loss: 3.5951 | LR: 3.84e-04 | 1885.5ms/step | 33.9 samples/s
+Step 3280/10001 | Loss: 3.6029 | LR: 3.83e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3290/10001 | Loss: 3.5837 | LR: 3.83e-04 | 1881.5ms/step | 34.0 samples/s
+Step 3300/10001 | Loss: 3.6098 | LR: 3.82e-04 | 1882.5ms/step | 34.0 samples/s
+Step 3310/10001 | Loss: 3.5930 | LR: 3.81e-04 | 1928.0ms/step | 33.2 samples/s
+Step 3320/10001 | Loss: 3.5842 | LR: 3.80e-04 | 1879.4ms/step | 34.1 samples/s
+Step 3330/10001 | Loss: 3.5548 | LR: 3.80e-04 | 1880.0ms/step | 34.0 samples/s
+Step 3340/10001 | Loss: 3.5681 | LR: 3.79e-04 | 1882.4ms/step | 34.0 samples/s
+Step 3350/10001 | Loss: 3.5678 | LR: 3.78e-04 | 1883.0ms/step | 34.0 samples/s
+Step 3360/10001 | Loss: 3.5747 | LR: 3.78e-04 | 1880.6ms/step | 34.0 samples/s
+Step 3370/10001 | Loss: 3.5882 | LR: 3.77e-04 | 1883.1ms/step | 34.0 samples/s
+Step 3380/10001 | Loss: 3.5655 | LR: 3.76e-04 | 1881.5ms/step | 34.0 samples/s
+Step 3390/10001 | Loss: 3.5738 | LR: 3.76e-04 | 1882.0ms/step | 34.0 samples/s
+Step 3400/10001 | Loss: 3.5534 | LR: 3.75e-04 | 1881.3ms/step | 34.0 samples/s
+Step 3410/10001 | Loss: 3.5810 | LR: 3.74e-04 | 1882.7ms/step | 34.0 samples/s
+Step 3420/10001 | Loss: 3.5694 | LR: 3.74e-04 | 1879.4ms/step | 34.1 samples/s
+Step 3430/10001 | Loss: 3.5514 | LR: 3.73e-04 | 1880.0ms/step | 34.0 samples/s
+Step 3440/10001 | Loss: 3.5604 | LR: 3.72e-04 | 1880.0ms/step | 34.0 samples/s
+Step 3450/10001 | Loss: 3.5799 | LR: 3.72e-04 | 1880.5ms/step | 34.0 samples/s
+Step 3460/10001 | Loss: 3.5825 | LR: 3.71e-04 | 1881.7ms/step | 34.0 samples/s
+Step 3470/10001 | Loss: 3.5563 | LR: 3.70e-04 | 1881.7ms/step | 34.0 samples/s
+Step 3480/10001 | Loss: 3.5628 | LR: 3.69e-04 | 1881.5ms/step | 34.0 samples/s
+Step 3490/10001 | Loss: 3.5602 | LR: 3.69e-04 | 1885.6ms/step | 33.9 samples/s
+Step 3500/10001 | Loss: 3.5404 | LR: 3.68e-04 | 1883.5ms/step | 34.0 samples/s
+Validation at step 3500: Loss = 3.5728, Perplexity = 35.61
+Step 3510/10001 | Loss: 3.5390 | LR: 3.67e-04 | 32063.4ms/step | 2.0 samples/s
+Step 3520/10001 | Loss: 3.5339 | LR: 3.67e-04 | 1881.8ms/step | 34.0 samples/s
+Step 3530/10001 | Loss: 3.5952 | LR: 3.66e-04 | 1883.9ms/step | 34.0 samples/s
+Step 3540/10001 | Loss: 3.5698 | LR: 3.65e-04 | 1883.4ms/step | 34.0 samples/s
+Step 3550/10001 | Loss: 3.5778 | LR: 3.65e-04 | 1883.5ms/step | 34.0 samples/s
+Step 3560/10001 | Loss: 3.5697 | LR: 3.64e-04 | 1881.9ms/step | 34.0 samples/s
+Step 3570/10001 | Loss: 3.5567 | LR: 3.63e-04 | 1881.4ms/step | 34.0 samples/s
+Step 3580/10001 | Loss: 3.5689 | LR: 3.62e-04 | 1883.5ms/step | 34.0 samples/s
+Step 3590/10001 | Loss: 3.5432 | LR: 3.62e-04 | 1884.6ms/step | 34.0 samples/s
+Step 3600/10001 | Loss: 3.5777 | LR: 3.61e-04 | 1884.4ms/step | 34.0 samples/s
+Step 3610/10001 | Loss: 3.5459 | LR: 3.60e-04 | 1883.1ms/step | 34.0 samples/s
+Step 3620/10001 | Loss: 3.5676 | LR: 3.60e-04 | 1883.1ms/step | 34.0 samples/s
+Step 3630/10001 | Loss: 3.5371 | LR: 3.59e-04 | 1881.3ms/step | 34.0 samples/s
+Step 3640/10001 | Loss: 3.5695 | LR: 3.58e-04 | 1880.6ms/step | 34.0 samples/s
+Step 3650/10001 | Loss: 3.5709 | LR: 3.57e-04 | 1883.4ms/step | 34.0 samples/s
+Step 3660/10001 | Loss: 3.5479 | LR: 3.57e-04 | 1882.5ms/step | 34.0 samples/s
+Step 3670/10001 | Loss: 3.5390 | LR: 3.56e-04 | 1879.5ms/step | 34.1 samples/s
+Step 3680/10001 | Loss: 3.5267 | LR: 3.55e-04 | 1881.7ms/step | 34.0 samples/s
+Step 3690/10001 | Loss: 3.5735 | LR: 3.55e-04 | 1880.8ms/step | 34.0 samples/s
+Step 3700/10001 | Loss: 3.5517 | LR: 3.54e-04 | 1882.6ms/step | 34.0 samples/s
+Step 3710/10001 | Loss: 3.5585 | LR: 3.53e-04 | 1883.6ms/step | 34.0 samples/s
+Step 3720/10001 | Loss: 3.5195 | LR: 3.52e-04 | 1883.3ms/step | 34.0 samples/s
+Step 3730/10001 | Loss: 3.5459 | LR: 3.52e-04 | 1883.6ms/step | 34.0 samples/s
+Step 3740/10001 | Loss: 3.5194 | LR: 3.51e-04 | 1882.4ms/step | 34.0 samples/s
+Step 3750/10001 | Loss: 3.5442 | LR: 3.50e-04 | 1880.0ms/step | 34.0 samples/s
+Validation at step 3750: Loss = 3.5438, Perplexity = 34.60
+Step 3760/10001 | Loss: 3.5882 | LR: 3.50e-04 | 32061.5ms/step | 2.0 samples/s
+Step 3770/10001 | Loss: 3.5606 | LR: 3.49e-04 | 1882.9ms/step | 34.0 samples/s
+Step 3780/10001 | Loss: 3.5746 | LR: 3.48e-04 | 1881.7ms/step | 34.0 samples/s
+Step 3790/10001 | Loss: 3.5176 | LR: 3.47e-04 | 1930.0ms/step | 33.2 samples/s
+Step 3800/10001 | Loss: 3.5299 | LR: 3.47e-04 | 1879.8ms/step | 34.0 samples/s
+Step 3810/10001 | Loss: 3.5318 | LR: 3.46e-04 | 1882.1ms/step | 34.0 samples/s
+Step 3820/10001 | Loss: 3.5015 | LR: 3.45e-04 | 1880.0ms/step | 34.0 samples/s
+Step 3830/10001 | Loss: 3.5414 | LR: 3.44e-04 | 1881.5ms/step | 34.0 samples/s
+Step 3840/10001 | Loss: 3.5551 | LR: 3.44e-04 | 1880.3ms/step | 34.0 samples/s
+Step 3850/10001 | Loss: 3.5279 | LR: 3.43e-04 | 1881.8ms/step | 34.0 samples/s
+Step 3860/10001 | Loss: 3.5315 | LR: 3.42e-04 | 1881.9ms/step | 34.0 samples/s
+Step 3870/10001 | Loss: 3.5203 | LR: 3.41e-04 | 1881.5ms/step | 34.0 samples/s
+Step 3880/10001 | Loss: 3.5619 | LR: 3.41e-04 | 1880.9ms/step | 34.0 samples/s
+Step 3890/10001 | Loss: 3.5358 | LR: 3.40e-04 | 1879.9ms/step | 34.0 samples/s
+Step 3900/10001 | Loss: 3.5231 | LR: 3.39e-04 | 1881.8ms/step | 34.0 samples/s
+Step 3910/10001 | Loss: 3.5473 | LR: 3.39e-04 | 1882.2ms/step | 34.0 samples/s
+Step 3920/10001 | Loss: 3.5182 | LR: 3.38e-04 | 1880.5ms/step | 34.0 samples/s
+Step 3930/10001 | Loss: 3.4888 | LR: 3.37e-04 | 1880.9ms/step | 34.0 samples/s
+Step 3940/10001 | Loss: 3.5196 | LR: 3.36e-04 | 1880.5ms/step | 34.0 samples/s
+Step 3950/10001 | Loss: 3.5209 | LR: 3.36e-04 | 1881.0ms/step | 34.0 samples/s
+Step 3960/10001 | Loss: 3.5322 | LR: 3.35e-04 | 1882.9ms/step | 34.0 samples/s
+Step 3970/10001 | Loss: 3.5221 | LR: 3.34e-04 | 1882.6ms/step | 34.0 samples/s
+Step 3980/10001 | Loss: 3.5455 | LR: 3.33e-04 | 1880.0ms/step | 34.0 samples/s
+Step 3990/10001 | Loss: 3.5111 | LR: 3.33e-04 | 1878.9ms/step | 34.1 samples/s
+Step 4000/10001 | Loss: 3.4936 | LR: 3.32e-04 | 1880.0ms/step | 34.0 samples/s
+Validation at step 4000: Loss = 3.5192, Perplexity = 33.76
+Step 4010/10001 | Loss: 3.5082 | LR: 3.31e-04 | 32028.4ms/step | 2.0 samples/s
+Step 4020/10001 | Loss: 3.5135 | LR: 3.30e-04 | 1882.7ms/step | 34.0 samples/s
+Step 4030/10001 | Loss: 3.4834 | LR: 3.30e-04 | 1882.4ms/step | 34.0 samples/s
+Step 4040/10001 | Loss: 3.5306 | LR: 3.29e-04 | 1885.7ms/step | 33.9 samples/s
+Step 4050/10001 | Loss: 3.5159 | LR: 3.28e-04 | 1879.2ms/step | 34.1 samples/s
+Step 4060/10001 | Loss: 3.5002 | LR: 3.27e-04 | 1879.5ms/step | 34.1 samples/s
+Step 4070/10001 | Loss: 3.5034 | LR: 3.27e-04 | 1879.8ms/step | 34.0 samples/s
+Step 4080/10001 | Loss: 3.5230 | LR: 3.26e-04 | 1879.7ms/step | 34.0 samples/s
+Step 4090/10001 | Loss: 3.4941 | LR: 3.25e-04 | 1877.8ms/step | 34.1 samples/s
+Step 4100/10001 | Loss: 3.4952 | LR: 3.24e-04 | 1877.8ms/step | 34.1 samples/s
+Step 4110/10001 | Loss: 3.5034 | LR: 3.24e-04 | 1881.4ms/step | 34.0 samples/s
+Step 4120/10001 | Loss: 3.5352 | LR: 3.23e-04 | 1878.9ms/step | 34.1 samples/s
+Step 4130/10001 | Loss: 3.5042 | LR: 3.22e-04 | 1878.5ms/step | 34.1 samples/s
+Step 4140/10001 | Loss: 3.4913 | LR: 3.21e-04 | 1878.6ms/step | 34.1 samples/s
+Step 4150/10001 | Loss: 3.4972 | LR: 3.20e-04 | 1878.4ms/step | 34.1 samples/s
+Step 4160/10001 | Loss: 3.4879 | LR: 3.20e-04 | 1879.0ms/step | 34.1 samples/s
+Step 4170/10001 | Loss: 3.5148 | LR: 3.19e-04 | 1878.0ms/step | 34.1 samples/s
+Step 4180/10001 | Loss: 3.5274 | LR: 3.18e-04 | 1878.2ms/step | 34.1 samples/s
+Step 4190/10001 | Loss: 3.4814 | LR: 3.17e-04 | 1877.9ms/step | 34.1 samples/s
+Step 4200/10001 | Loss: 3.5218 | LR: 3.17e-04 | 1878.2ms/step | 34.1 samples/s
+Step 4210/10001 | Loss: 3.5098 | LR: 3.16e-04 | 1878.9ms/step | 34.1 samples/s
+Step 4220/10001 | Loss: 3.4995 | LR: 3.15e-04 | 1878.8ms/step | 34.1 samples/s
+Step 4230/10001 | Loss: 3.5030 | LR: 3.14e-04 | 1879.5ms/step | 34.1 samples/s
+Step 4240/10001 | Loss: 3.5062 | LR: 3.14e-04 | 1880.5ms/step | 34.0 samples/s
+Step 4250/10001 | Loss: 3.4742 | LR: 3.13e-04 | 1878.5ms/step | 34.1 samples/s
+Validation at step 4250: Loss = 3.4952, Perplexity = 32.96
+Step 4260/10001 | Loss: 3.4784 | LR: 3.12e-04 | 31973.7ms/step | 2.0 samples/s
+Step 4270/10001 | Loss: 3.4962 | LR: 3.11e-04 | 1884.0ms/step | 34.0 samples/s
+Step 4280/10001 | Loss: 3.4602 | LR: 3.11e-04 | 1885.8ms/step | 33.9 samples/s
+Step 4290/10001 | Loss: 3.4812 | LR: 3.10e-04 | 1887.6ms/step | 33.9 samples/s
+Step 4300/10001 | Loss: 3.4629 | LR: 3.09e-04 | 1886.1ms/step | 33.9 samples/s
+Step 4310/10001 | Loss: 3.4893 | LR: 3.08e-04 | 1886.4ms/step | 33.9 samples/s
+Step 4320/10001 | Loss: 3.5170 | LR: 3.07e-04 | 1885.4ms/step | 33.9 samples/s
+Step 4330/10001 | Loss: 3.4948 | LR: 3.07e-04 | 1884.0ms/step | 34.0 samples/s
+Step 4340/10001 | Loss: 3.4841 | LR: 3.06e-04 | 1886.2ms/step | 33.9 samples/s
+Step 4350/10001 | Loss: 3.4840 | LR: 3.05e-04 | 1887.6ms/step | 33.9 samples/s
+Step 4360/10001 | Loss: 3.4586 | LR: 3.04e-04 | 1888.3ms/step | 33.9 samples/s
+Step 4370/10001 | Loss: 3.4783 | LR: 3.04e-04 | 1885.8ms/step | 33.9 samples/s
+Step 4380/10001 | Loss: 3.4992 | LR: 3.03e-04 | 1885.1ms/step | 34.0 samples/s
+Step 4390/10001 | Loss: 3.5043 | LR: 3.02e-04 | 1885.1ms/step | 34.0 samples/s
+Step 4400/10001 | Loss: 3.4782 | LR: 3.01e-04 | 1885.9ms/step | 33.9 samples/s
+Step 4410/10001 | Loss: 3.4818 | LR: 3.00e-04 | 1888.1ms/step | 33.9 samples/s
+Step 4420/10001 | Loss: 3.4440 | LR: 3.00e-04 | 1886.8ms/step | 33.9 samples/s
+Step 4430/10001 | Loss: 3.5059 | LR: 2.99e-04 | 1885.1ms/step | 34.0 samples/s
+Step 4440/10001 | Loss: 3.4485 | LR: 2.98e-04 | 1888.2ms/step | 33.9 samples/s
+Step 4450/10001 | Loss: 3.4844 | LR: 2.97e-04 | 1888.0ms/step | 33.9 samples/s
+Step 4460/10001 | Loss: 3.4869 | LR: 2.97e-04 | 1887.7ms/step | 33.9 samples/s
+Step 4470/10001 | Loss: 3.4694 | LR: 2.96e-04 | 1883.6ms/step | 34.0 samples/s
+Step 4480/10001 | Loss: 3.4556 | LR: 2.95e-04 | 1884.0ms/step | 34.0 samples/s
+Step 4490/10001 | Loss: 3.5095 | LR: 2.94e-04 | 1886.9ms/step | 33.9 samples/s
+Step 4500/10001 | Loss: 3.4611 | LR: 2.93e-04 | 1885.0ms/step | 34.0 samples/s
+Validation at step 4500: Loss = 3.4745, Perplexity = 32.28
+Step 4510/10001 | Loss: 3.4607 | LR: 2.93e-04 | 32104.9ms/step | 2.0 samples/s
+Step 4520/10001 | Loss: 3.4788 | LR: 2.92e-04 | 1887.7ms/step | 33.9 samples/s
+Step 4530/10001 | Loss: 3.4490 | LR: 2.91e-04 | 1885.6ms/step | 33.9 samples/s
+Step 4540/10001 | Loss: 3.4754 | LR: 2.90e-04 | 1886.4ms/step | 33.9 samples/s
+Step 4550/10001 | Loss: 3.4773 | LR: 2.90e-04 | 1883.6ms/step | 34.0 samples/s
+Step 4560/10001 | Loss: 3.4732 | LR: 2.89e-04 | 1888.2ms/step | 33.9 samples/s
+Step 4570/10001 | Loss: 3.4648 | LR: 2.88e-04 | 1886.6ms/step | 33.9 samples/s
+Step 4580/10001 | Loss: 3.4744 | LR: 2.87e-04 | 1885.0ms/step | 34.0 samples/s
+Step 4590/10001 | Loss: 3.4590 | LR: 2.86e-04 | 1888.5ms/step | 33.9 samples/s
+Step 4600/10001 | Loss: 3.4473 | LR: 2.86e-04 | 1887.0ms/step | 33.9 samples/s
+Step 4610/10001 | Loss: 3.4318 | LR: 2.85e-04 | 1887.9ms/step | 33.9 samples/s
+Step 4620/10001 | Loss: 3.4698 | LR: 2.84e-04 | 1888.8ms/step | 33.9 samples/s
+Step 4630/10001 | Loss: 3.4596 | LR: 2.83e-04 | 1886.3ms/step | 33.9 samples/s
+Step 4640/10001 | Loss: 3.4807 | LR: 2.82e-04 | 1887.7ms/step | 33.9 samples/s
+Step 4650/10001 | Loss: 3.4708 | LR: 2.82e-04 | 1887.9ms/step | 33.9 samples/s
+Step 4660/10001 | Loss: 3.4607 | LR: 2.81e-04 | 1887.3ms/step | 33.9 samples/s
+Step 4670/10001 | Loss: 3.4296 | LR: 2.80e-04 | 1888.0ms/step | 33.9 samples/s
+Step 4680/10001 | Loss: 3.4407 | LR: 2.79e-04 | 1888.1ms/step | 33.9 samples/s
+Step 4690/10001 | Loss: 3.4324 | LR: 2.79e-04 | 1887.2ms/step | 33.9 samples/s
+Step 4700/10001 | Loss: 3.4200 | LR: 2.78e-04 | 1885.7ms/step | 33.9 samples/s
+Step 4710/10001 | Loss: 3.4501 | LR: 2.77e-04 | 1885.4ms/step | 33.9 samples/s
+Step 4720/10001 | Loss: 3.5005 | LR: 2.76e-04 | 1885.7ms/step | 33.9 samples/s
+Step 4730/10001 | Loss: 3.4579 | LR: 2.75e-04 | 1885.3ms/step | 33.9 samples/s
+Step 4740/10001 | Loss: 3.4415 | LR: 2.75e-04 | 1933.6ms/step | 33.1 samples/s
+Step 4750/10001 | Loss: 3.4427 | LR: 2.74e-04 | 1885.5ms/step | 33.9 samples/s
+Validation at step 4750: Loss = 3.4543, Perplexity = 31.64
+Step 4760/10001 | Loss: 3.4735 | LR: 2.73e-04 | 32108.5ms/step | 2.0 samples/s
+Step 4770/10001 | Loss: 3.4627 | LR: 2.72e-04 | 1886.3ms/step | 33.9 samples/s
+Step 4780/10001 | Loss: 3.4680 | LR: 2.71e-04 | 1887.9ms/step | 33.9 samples/s
+Step 4790/10001 | Loss: 3.4510 | LR: 2.71e-04 | 1888.2ms/step | 33.9 samples/s
+Step 4800/10001 | Loss: 3.4431 | LR: 2.70e-04 | 1887.3ms/step | 33.9 samples/s
+Step 4810/10001 | Loss: 3.4439 | LR: 2.69e-04 | 1888.3ms/step | 33.9 samples/s
+Step 4820/10001 | Loss: 3.4412 | LR: 2.68e-04 | 1887.3ms/step | 33.9 samples/s
+Step 4830/10001 | Loss: 3.4257 | LR: 2.67e-04 | 1886.5ms/step | 33.9 samples/s
+Step 4840/10001 | Loss: 3.4780 | LR: 2.67e-04 | 1887.5ms/step | 33.9 samples/s
+Step 4850/10001 | Loss: 3.4530 | LR: 2.66e-04 | 1887.2ms/step | 33.9 samples/s
+Step 4860/10001 | Loss: 3.4556 | LR: 2.65e-04 | 1887.5ms/step | 33.9 samples/s
+Step 4870/10001 | Loss: 3.4684 | LR: 2.64e-04 | 1886.9ms/step | 33.9 samples/s
+Step 4880/10001 | Loss: 3.4263 | LR: 2.64e-04 | 1885.2ms/step | 33.9 samples/s
+Step 4890/10001 | Loss: 3.4503 | LR: 2.63e-04 | 1884.9ms/step | 34.0 samples/s
+Step 4900/10001 | Loss: 3.4419 | LR: 2.62e-04 | 1885.6ms/step | 33.9 samples/s
+Step 4910/10001 | Loss: 3.4484 | LR: 2.61e-04 | 1886.0ms/step | 33.9 samples/s
+Step 4920/10001 | Loss: 3.4133 | LR: 2.60e-04 | 1887.2ms/step | 33.9 samples/s
+Step 4930/10001 | Loss: 3.4074 | LR: 2.60e-04 | 1887.1ms/step | 33.9 samples/s
+Step 4940/10001 | Loss: 3.4395 | LR: 2.59e-04 | 1884.5ms/step | 34.0 samples/s
+Step 4950/10001 | Loss: 3.4788 | LR: 2.58e-04 | 1885.5ms/step | 33.9 samples/s
+Step 4960/10001 | Loss: 3.4512 | LR: 2.57e-04 | 1884.1ms/step | 34.0 samples/s
+Step 4970/10001 | Loss: 3.4259 | LR: 2.56e-04 | 1887.1ms/step | 33.9 samples/s
+Step 4980/10001 | Loss: 3.4419 | LR: 2.56e-04 | 1888.1ms/step | 33.9 samples/s
+Step 4990/10001 | Loss: 3.4434 | LR: 2.55e-04 | 1889.2ms/step | 33.9 samples/s
+Step 5000/10001 | Loss: 3.4342 | LR: 2.54e-04 | 1886.9ms/step | 33.9 samples/s
+Validation at step 5000: Loss = 3.4360, Perplexity = 31.06
+Saved checkpoint to checkpoints/run_20260430_020658/checkpoint-5000
+Step 5010/10001 | Loss: 3.4650 | LR: 2.53e-04 | 32604.4ms/step | 2.0 samples/s
+Step 5020/10001 | Loss: 3.4162 | LR: 2.52e-04 | 1882.8ms/step | 34.0 samples/s
+Step 5030/10001 | Loss: 3.4002 | LR: 2.52e-04 | 1883.9ms/step | 34.0 samples/s
+Step 5040/10001 | Loss: 3.4526 | LR: 2.51e-04 | 1885.3ms/step | 33.9 samples/s
+Step 5050/10001 | Loss: 3.4461 | LR: 2.50e-04 | 1883.6ms/step | 34.0 samples/s
+Step 5060/10001 | Loss: 3.4364 | LR: 2.49e-04 | 1883.1ms/step | 34.0 samples/s
+Step 5070/10001 | Loss: 3.4474 | LR: 2.48e-04 | 1882.4ms/step | 34.0 samples/s
+Step 5080/10001 | Loss: 3.4254 | LR: 2.48e-04 | 1884.1ms/step | 34.0 samples/s
+Step 5090/10001 | Loss: 3.4429 | LR: 2.47e-04 | 1882.1ms/step | 34.0 samples/s
+Step 5100/10001 | Loss: 3.4088 | LR: 2.46e-04 | 1883.4ms/step | 34.0 samples/s
+Step 5110/10001 | Loss: 3.4311 | LR: 2.45e-04 | 1884.5ms/step | 34.0 samples/s
+Step 5120/10001 | Loss: 3.4278 | LR: 2.44e-04 | 1883.4ms/step | 34.0 samples/s
+Step 5130/10001 | Loss: 3.4289 | LR: 2.44e-04 | 1882.2ms/step | 34.0 samples/s
+Step 5140/10001 | Loss: 3.4527 | LR: 2.43e-04 | 1883.8ms/step | 34.0 samples/s
+Step 5150/10001 | Loss: 3.4212 | LR: 2.42e-04 | 1884.9ms/step | 34.0 samples/s
+Step 5160/10001 | Loss: 3.4332 | LR: 2.41e-04 | 1883.8ms/step | 34.0 samples/s
+Step 5170/10001 | Loss: 3.4433 | LR: 2.41e-04 | 1883.3ms/step | 34.0 samples/s
+Step 5180/10001 | Loss: 3.4414 | LR: 2.40e-04 | 1884.6ms/step | 34.0 samples/s
+Step 5190/10001 | Loss: 3.4194 | LR: 2.39e-04 | 1885.9ms/step | 33.9 samples/s
+Step 5200/10001 | Loss: 3.4200 | LR: 2.38e-04 | 1885.1ms/step | 33.9 samples/s
+Step 5210/10001 | Loss: 3.4358 | LR: 2.37e-04 | 1935.9ms/step | 33.1 samples/s
+Step 5220/10001 | Loss: 3.4237 | LR: 2.37e-04 | 1882.7ms/step | 34.0 samples/s
+Step 5230/10001 | Loss: 3.4342 | LR: 2.36e-04 | 1878.5ms/step | 34.1 samples/s
+Step 5240/10001 | Loss: 3.4271 | LR: 2.35e-04 | 1878.4ms/step | 34.1 samples/s
+Step 5250/10001 | Loss: 3.3928 | LR: 2.34e-04 | 1880.2ms/step | 34.0 samples/s
+Validation at step 5250: Loss = 3.4193, Perplexity = 30.55
+Step 5260/10001 | Loss: 3.4096 | LR: 2.33e-04 | 31983.9ms/step | 2.0 samples/s
+Step 5270/10001 | Loss: 3.4164 | LR: 2.33e-04 | 1880.0ms/step | 34.0 samples/s
+Step 5280/10001 | Loss: 3.4022 | LR: 2.32e-04 | 1880.5ms/step | 34.0 samples/s
+Step 5290/10001 | Loss: 3.3982 | LR: 2.31e-04 | 1878.2ms/step | 34.1 samples/s
+Step 5300/10001 | Loss: 3.4560 | LR: 2.30e-04 | 1877.2ms/step | 34.1 samples/s
+Step 5310/10001 | Loss: 3.4097 | LR: 2.29e-04 | 1877.5ms/step | 34.1 samples/s
+Step 5320/10001 | Loss: 3.3913 | LR: 2.29e-04 | 1878.1ms/step | 34.1 samples/s
+Step 5330/10001 | Loss: 3.3977 | LR: 2.28e-04 | 1878.0ms/step | 34.1 samples/s
+Step 5340/10001 | Loss: 3.3976 | LR: 2.27e-04 | 1878.4ms/step | 34.1 samples/s
+Step 5350/10001 | Loss: 3.4270 | LR: 2.26e-04 | 1879.4ms/step | 34.1 samples/s
+Step 5360/10001 | Loss: 3.4096 | LR: 2.25e-04 | 1878.3ms/step | 34.1 samples/s
+Step 5370/10001 | Loss: 3.4172 | LR: 2.25e-04 | 1877.6ms/step | 34.1 samples/s
+Step 5380/10001 | Loss: 3.3975 | LR: 2.24e-04 | 1878.3ms/step | 34.1 samples/s
+Step 5390/10001 | Loss: 3.3990 | LR: 2.23e-04 | 1879.7ms/step | 34.0 samples/s
+Step 5400/10001 | Loss: 3.4479 | LR: 2.22e-04 | 1877.9ms/step | 34.1 samples/s
+Step 5410/10001 | Loss: 3.3990 | LR: 2.22e-04 | 1876.9ms/step | 34.1 samples/s
+Step 5420/10001 | Loss: 3.4140 | LR: 2.21e-04 | 1878.7ms/step | 34.1 samples/s
+Step 5430/10001 | Loss: 3.4213 | LR: 2.20e-04 | 1878.6ms/step | 34.1 samples/s
+Step 5440/10001 | Loss: 3.4292 | LR: 2.19e-04 | 1878.1ms/step | 34.1 samples/s
+Step 5450/10001 | Loss: 3.4083 | LR: 2.18e-04 | 1878.4ms/step | 34.1 samples/s
+Step 5460/10001 | Loss: 3.4247 | LR: 2.18e-04 | 1879.8ms/step | 34.0 samples/s
+Step 5470/10001 | Loss: 3.4427 | LR: 2.17e-04 | 1878.2ms/step | 34.1 samples/s
+Step 5480/10001 | Loss: 3.4225 | LR: 2.16e-04 | 1877.7ms/step | 34.1 samples/s
+Step 5490/10001 | Loss: 3.3956 | LR: 2.15e-04 | 1879.4ms/step | 34.1 samples/s
+Step 5500/10001 | Loss: 3.3967 | LR: 2.14e-04 | 1878.0ms/step | 34.1 samples/s
+Validation at step 5500: Loss = 3.4044, Perplexity = 30.10
+Step 5510/10001 | Loss: 3.4029 | LR: 2.14e-04 | 31926.2ms/step | 2.0 samples/s
+Step 5520/10001 | Loss: 3.4198 | LR: 2.13e-04 | 1877.9ms/step | 34.1 samples/s
+Step 5530/10001 | Loss: 3.4238 | LR: 2.12e-04 | 1880.4ms/step | 34.0 samples/s
+Step 5540/10001 | Loss: 3.4222 | LR: 2.11e-04 | 1880.5ms/step | 34.0 samples/s
+Step 5550/10001 | Loss: 3.3988 | LR: 2.11e-04 | 1880.6ms/step | 34.0 samples/s
+Step 5560/10001 | Loss: 3.4058 | LR: 2.10e-04 | 1876.9ms/step | 34.1 samples/s
+Step 5570/10001 | Loss: 3.3741 | LR: 2.09e-04 | 1883.6ms/step | 34.0 samples/s
+Step 5580/10001 | Loss: 3.4085 | LR: 2.08e-04 | 1880.0ms/step | 34.0 samples/s
+Step 5590/10001 | Loss: 3.4150 | LR: 2.07e-04 | 1879.3ms/step | 34.1 samples/s
+Step 5600/10001 | Loss: 3.4019 | LR: 2.07e-04 | 1878.8ms/step | 34.1 samples/s
+Step 5610/10001 | Loss: 3.4242 | LR: 2.06e-04 | 1880.3ms/step | 34.0 samples/s
+Step 5620/10001 | Loss: 3.3939 | LR: 2.05e-04 | 1880.5ms/step | 34.0 samples/s
+Step 5630/10001 | Loss: 3.3840 | LR: 2.04e-04 | 1880.8ms/step | 34.0 samples/s
+Step 5640/10001 | Loss: 3.3749 | LR: 2.04e-04 | 1883.8ms/step | 34.0 samples/s
+Step 5650/10001 | Loss: 3.3965 | LR: 2.03e-04 | 1880.0ms/step | 34.0 samples/s
+Step 5660/10001 | Loss: 3.4114 | LR: 2.02e-04 | 1882.9ms/step | 34.0 samples/s
+Step 5670/10001 | Loss: 3.4140 | LR: 2.01e-04 | 1879.9ms/step | 34.0 samples/s
+Step 5680/10001 | Loss: 3.3974 | LR: 2.00e-04 | 1880.5ms/step | 34.0 samples/s
+Step 5690/10001 | Loss: 3.4122 | LR: 2.00e-04 | 1929.7ms/step | 33.2 samples/s
+Step 5700/10001 | Loss: 3.3964 | LR: 1.99e-04 | 1882.8ms/step | 34.0 samples/s
+Step 5710/10001 | Loss: 3.3842 | LR: 1.98e-04 | 1879.7ms/step | 34.0 samples/s
+Step 5720/10001 | Loss: 3.3593 | LR: 1.97e-04 | 1879.9ms/step | 34.0 samples/s
+Step 5730/10001 | Loss: 3.3657 | LR: 1.97e-04 | 1882.3ms/step | 34.0 samples/s
+Step 5740/10001 | Loss: 3.3899 | LR: 1.96e-04 | 1880.3ms/step | 34.0 samples/s
+Step 5750/10001 | Loss: 3.4132 | LR: 1.95e-04 | 1879.6ms/step | 34.1 samples/s
+Validation at step 5750: Loss = 3.3908, Perplexity = 29.69
+Step 5760/10001 | Loss: 3.3785 | LR: 1.94e-04 | 32040.3ms/step | 2.0 samples/s
+Step 5770/10001 | Loss: 3.3985 | LR: 1.93e-04 | 1883.0ms/step | 34.0 samples/s
+Step 5780/10001 | Loss: 3.4069 | LR: 1.93e-04 | 1881.0ms/step | 34.0 samples/s
+Step 5790/10001 | Loss: 3.3966 | LR: 1.92e-04 | 1881.9ms/step | 34.0 samples/s
+Step 5800/10001 | Loss: 3.3456 | LR: 1.91e-04 | 1883.1ms/step | 34.0 samples/s
+Step 5810/10001 | Loss: 3.4011 | LR: 1.90e-04 | 1882.5ms/step | 34.0 samples/s
+Step 5820/10001 | Loss: 3.3595 | LR: 1.90e-04 | 1879.5ms/step | 34.1 samples/s
+Step 5830/10001 | Loss: 3.3928 | LR: 1.89e-04 | 1880.3ms/step | 34.0 samples/s
+Step 5840/10001 | Loss: 3.3958 | LR: 1.88e-04 | 1882.2ms/step | 34.0 samples/s
+Step 5850/10001 | Loss: 3.3987 | LR: 1.87e-04 | 1882.9ms/step | 34.0 samples/s
+Step 5860/10001 | Loss: 3.3801 | LR: 1.86e-04 | 1885.0ms/step | 34.0 samples/s
+Step 5870/10001 | Loss: 3.3801 | LR: 1.86e-04 | 1882.7ms/step | 34.0 samples/s
+Step 5880/10001 | Loss: 3.3970 | LR: 1.85e-04 | 1883.7ms/step | 34.0 samples/s
+Step 5890/10001 | Loss: 3.3750 | LR: 1.84e-04 | 1884.6ms/step | 34.0 samples/s
+Step 5900/10001 | Loss: 3.3679 | LR: 1.83e-04 | 1882.5ms/step | 34.0 samples/s
+Step 5910/10001 | Loss: 3.3557 | LR: 1.83e-04 | 1880.4ms/step | 34.0 samples/s
+Step 5920/10001 | Loss: 3.3643 | LR: 1.82e-04 | 1880.0ms/step | 34.0 samples/s
+Step 5930/10001 | Loss: 3.3873 | LR: 1.81e-04 | 1879.6ms/step | 34.1 samples/s
+Step 5940/10001 | Loss: 3.4046 | LR: 1.80e-04 | 1882.5ms/step | 34.0 samples/s
+Step 5950/10001 | Loss: 3.3760 | LR: 1.80e-04 | 1880.2ms/step | 34.0 samples/s
+Step 5960/10001 | Loss: 3.3856 | LR: 1.79e-04 | 1879.4ms/step | 34.1 samples/s
+Step 5970/10001 | Loss: 3.3718 | LR: 1.78e-04 | 1882.2ms/step | 34.0 samples/s
+Step 5980/10001 | Loss: 3.3820 | LR: 1.77e-04 | 1881.0ms/step | 34.0 samples/s
+Step 5990/10001 | Loss: 3.4284 | LR: 1.77e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6000/10001 | Loss: 3.3575 | LR: 1.76e-04 | 1882.5ms/step | 34.0 samples/s
+Validation at step 6000: Loss = 3.3788, Perplexity = 29.33
+Step 6010/10001 | Loss: 3.3687 | LR: 1.75e-04 | 32043.9ms/step | 2.0 samples/s
+Step 6020/10001 | Loss: 3.3540 | LR: 1.74e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6030/10001 | Loss: 3.3746 | LR: 1.74e-04 | 1879.4ms/step | 34.1 samples/s
+Step 6040/10001 | Loss: 3.3606 | LR: 1.73e-04 | 1880.0ms/step | 34.0 samples/s
+Step 6050/10001 | Loss: 3.3708 | LR: 1.72e-04 | 1884.8ms/step | 34.0 samples/s
+Step 6060/10001 | Loss: 3.3743 | LR: 1.71e-04 | 1883.3ms/step | 34.0 samples/s
+Step 6070/10001 | Loss: 3.3993 | LR: 1.71e-04 | 1882.0ms/step | 34.0 samples/s
+Step 6080/10001 | Loss: 3.3612 | LR: 1.70e-04 | 1880.0ms/step | 34.0 samples/s
+Step 6090/10001 | Loss: 3.3941 | LR: 1.69e-04 | 1880.6ms/step | 34.0 samples/s
+Step 6100/10001 | Loss: 3.4174 | LR: 1.68e-04 | 1881.4ms/step | 34.0 samples/s
+Step 6110/10001 | Loss: 3.3548 | LR: 1.68e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6120/10001 | Loss: 3.3403 | LR: 1.67e-04 | 1881.6ms/step | 34.0 samples/s
+Step 6130/10001 | Loss: 3.3642 | LR: 1.66e-04 | 1881.5ms/step | 34.0 samples/s
+Step 6140/10001 | Loss: 3.3733 | LR: 1.65e-04 | 1881.2ms/step | 34.0 samples/s
+Step 6150/10001 | Loss: 3.3966 | LR: 1.65e-04 | 1880.3ms/step | 34.0 samples/s
+Step 6160/10001 | Loss: 3.3585 | LR: 1.64e-04 | 1880.8ms/step | 34.0 samples/s
+Step 6170/10001 | Loss: 3.3671 | LR: 1.63e-04 | 1924.9ms/step | 33.2 samples/s
+Step 6180/10001 | Loss: 3.3764 | LR: 1.62e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6190/10001 | Loss: 3.3693 | LR: 1.62e-04 | 1880.4ms/step | 34.0 samples/s
+Step 6200/10001 | Loss: 3.3639 | LR: 1.61e-04 | 1878.7ms/step | 34.1 samples/s
+Step 6210/10001 | Loss: 3.3646 | LR: 1.60e-04 | 1881.4ms/step | 34.0 samples/s
+Step 6220/10001 | Loss: 3.3428 | LR: 1.59e-04 | 1881.4ms/step | 34.0 samples/s
+Step 6230/10001 | Loss: 3.3492 | LR: 1.59e-04 | 1880.3ms/step | 34.0 samples/s
+Step 6240/10001 | Loss: 3.3942 | LR: 1.58e-04 | 1880.3ms/step | 34.0 samples/s
+Step 6250/10001 | Loss: 3.3726 | LR: 1.57e-04 | 1882.0ms/step | 34.0 samples/s
+Validation at step 6250: Loss = 3.3681, Perplexity = 29.02
+Step 6260/10001 | Loss: 3.3794 | LR: 1.56e-04 | 32016.7ms/step | 2.0 samples/s
+Step 6270/10001 | Loss: 3.3577 | LR: 1.56e-04 | 1880.6ms/step | 34.0 samples/s
+Step 6280/10001 | Loss: 3.3417 | LR: 1.55e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6290/10001 | Loss: 3.3636 | LR: 1.54e-04 | 1879.3ms/step | 34.1 samples/s
+Step 6300/10001 | Loss: 3.3869 | LR: 1.53e-04 | 1880.7ms/step | 34.0 samples/s
+Step 6310/10001 | Loss: 3.3692 | LR: 1.53e-04 | 1880.7ms/step | 34.0 samples/s
+Step 6320/10001 | Loss: 3.3401 | LR: 1.52e-04 | 1882.7ms/step | 34.0 samples/s
+Step 6330/10001 | Loss: 3.3796 | LR: 1.51e-04 | 1884.6ms/step | 34.0 samples/s
+Step 6340/10001 | Loss: 3.3579 | LR: 1.51e-04 | 1879.7ms/step | 34.0 samples/s
+Step 6350/10001 | Loss: 3.3599 | LR: 1.50e-04 | 1880.9ms/step | 34.0 samples/s
+Step 6360/10001 | Loss: 3.3701 | LR: 1.49e-04 | 1878.7ms/step | 34.1 samples/s
+Step 6370/10001 | Loss: 3.3442 | LR: 1.48e-04 | 1880.2ms/step | 34.0 samples/s
+Step 6380/10001 | Loss: 3.3447 | LR: 1.48e-04 | 1881.0ms/step | 34.0 samples/s
+Step 6390/10001 | Loss: 3.3688 | LR: 1.47e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6400/10001 | Loss: 3.3655 | LR: 1.46e-04 | 1881.7ms/step | 34.0 samples/s
+Step 6410/10001 | Loss: 3.3775 | LR: 1.45e-04 | 1879.2ms/step | 34.1 samples/s
+Step 6420/10001 | Loss: 3.3699 | LR: 1.45e-04 | 1877.8ms/step | 34.1 samples/s
+Step 6430/10001 | Loss: 3.3517 | LR: 1.44e-04 | 1880.2ms/step | 34.0 samples/s
+Step 6440/10001 | Loss: 3.3757 | LR: 1.43e-04 | 1880.7ms/step | 34.0 samples/s
+Step 6450/10001 | Loss: 3.3462 | LR: 1.43e-04 | 1881.5ms/step | 34.0 samples/s
+Step 6460/10001 | Loss: 3.3681 | LR: 1.42e-04 | 1880.4ms/step | 34.0 samples/s
+Step 6470/10001 | Loss: 3.3533 | LR: 1.41e-04 | 1882.9ms/step | 34.0 samples/s
+Step 6480/10001 | Loss: 3.3862 | LR: 1.40e-04 | 1881.0ms/step | 34.0 samples/s
+Step 6490/10001 | Loss: 3.3493 | LR: 1.40e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6500/10001 | Loss: 3.3349 | LR: 1.39e-04 | 1879.5ms/step | 34.1 samples/s
+Validation at step 6500: Loss = 3.3587, Perplexity = 28.75
+Step 6510/10001 | Loss: 3.3447 | LR: 1.38e-04 | 32019.4ms/step | 2.0 samples/s
+Step 6520/10001 | Loss: 3.3447 | LR: 1.38e-04 | 1880.2ms/step | 34.0 samples/s
+Step 6530/10001 | Loss: 3.3807 | LR: 1.37e-04 | 1880.8ms/step | 34.0 samples/s
+Step 6540/10001 | Loss: 3.3671 | LR: 1.36e-04 | 1881.4ms/step | 34.0 samples/s
+Step 6550/10001 | Loss: 3.3912 | LR: 1.35e-04 | 1879.6ms/step | 34.0 samples/s
+Step 6560/10001 | Loss: 3.3586 | LR: 1.35e-04 | 1880.4ms/step | 34.0 samples/s
+Step 6570/10001 | Loss: 3.3190 | LR: 1.34e-04 | 1880.5ms/step | 34.0 samples/s
+Step 6580/10001 | Loss: 3.3669 | LR: 1.33e-04 | 1879.5ms/step | 34.1 samples/s
+Step 6590/10001 | Loss: 3.3555 | LR: 1.33e-04 | 1880.0ms/step | 34.0 samples/s
+Step 6600/10001 | Loss: 3.3484 | LR: 1.32e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6610/10001 | Loss: 3.3503 | LR: 1.31e-04 | 1881.9ms/step | 34.0 samples/s
+Step 6620/10001 | Loss: 3.3289 | LR: 1.31e-04 | 1881.2ms/step | 34.0 samples/s
+Step 6630/10001 | Loss: 3.3400 | LR: 1.30e-04 | 1879.8ms/step | 34.0 samples/s
+Step 6640/10001 | Loss: 3.3383 | LR: 1.29e-04 | 1880.0ms/step | 34.0 samples/s
+Step 6650/10001 | Loss: 3.3567 | LR: 1.28e-04 | 1928.5ms/step | 33.2 samples/s
+Step 6660/10001 | Loss: 3.3648 | LR: 1.28e-04 | 1883.1ms/step | 34.0 samples/s
+Step 6670/10001 | Loss: 3.3738 | LR: 1.27e-04 | 1885.0ms/step | 34.0 samples/s
+Step 6680/10001 | Loss: 3.3504 | LR: 1.26e-04 | 1881.2ms/step | 34.0 samples/s
+Step 6690/10001 | Loss: 3.3621 | LR: 1.26e-04 | 1881.2ms/step | 34.0 samples/s
+Step 6700/10001 | Loss: 3.3363 | LR: 1.25e-04 | 1880.1ms/step | 34.0 samples/s
+Step 6710/10001 | Loss: 3.3607 | LR: 1.24e-04 | 1878.0ms/step | 34.1 samples/s
+Step 6720/10001 | Loss: 3.3540 | LR: 1.24e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6730/10001 | Loss: 3.3396 | LR: 1.23e-04 | 1881.6ms/step | 34.0 samples/s
+Step 6740/10001 | Loss: 3.3320 | LR: 1.22e-04 | 1882.6ms/step | 34.0 samples/s
+Step 6750/10001 | Loss: 3.3502 | LR: 1.22e-04 | 1885.9ms/step | 33.9 samples/s
+Validation at step 6750: Loss = 3.3506, Perplexity = 28.52
+Step 6760/10001 | Loss: 3.3511 | LR: 1.21e-04 | 32036.5ms/step | 2.0 samples/s
+Step 6770/10001 | Loss: 3.3469 | LR: 1.20e-04 | 1884.4ms/step | 34.0 samples/s
+Step 6780/10001 | Loss: 3.3644 | LR: 1.20e-04 | 1880.5ms/step | 34.0 samples/s
+Step 6790/10001 | Loss: 3.3445 | LR: 1.19e-04 | 1880.6ms/step | 34.0 samples/s
+Step 6800/10001 | Loss: 3.3238 | LR: 1.18e-04 | 1882.1ms/step | 34.0 samples/s
+Step 6810/10001 | Loss: 3.3444 | LR: 1.18e-04 | 1881.3ms/step | 34.0 samples/s
+Step 6820/10001 | Loss: 3.3512 | LR: 1.17e-04 | 1880.0ms/step | 34.0 samples/s
+Step 6830/10001 | Loss: 3.3245 | LR: 1.16e-04 | 1882.6ms/step | 34.0 samples/s
+Step 6840/10001 | Loss: 3.3558 | LR: 1.16e-04 | 1883.5ms/step | 34.0 samples/s
+Step 6850/10001 | Loss: 3.3358 | LR: 1.15e-04 | 1883.9ms/step | 34.0 samples/s
+Step 6860/10001 | Loss: 3.3602 | LR: 1.14e-04 | 1882.9ms/step | 34.0 samples/s
+Step 6870/10001 | Loss: 3.3280 | LR: 1.14e-04 | 1882.6ms/step | 34.0 samples/s
+Step 6880/10001 | Loss: 3.3335 | LR: 1.13e-04 | 1882.1ms/step | 34.0 samples/s
+Step 6890/10001 | Loss: 3.3684 | LR: 1.12e-04 | 1881.1ms/step | 34.0 samples/s
+Step 6900/10001 | Loss: 3.3593 | LR: 1.12e-04 | 1881.2ms/step | 34.0 samples/s
+Step 6910/10001 | Loss: 3.3368 | LR: 1.11e-04 | 1882.1ms/step | 34.0 samples/s
+Step 6920/10001 | Loss: 3.3450 | LR: 1.10e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6930/10001 | Loss: 3.3321 | LR: 1.10e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6940/10001 | Loss: 3.3383 | LR: 1.09e-04 | 1880.9ms/step | 34.0 samples/s
+Step 6950/10001 | Loss: 3.3539 | LR: 1.08e-04 | 1882.5ms/step | 34.0 samples/s
+Step 6960/10001 | Loss: 3.3602 | LR: 1.08e-04 | 1879.9ms/step | 34.0 samples/s
+Step 6970/10001 | Loss: 3.3297 | LR: 1.07e-04 | 1880.2ms/step | 34.0 samples/s
+Step 6980/10001 | Loss: 3.3581 | LR: 1.06e-04 | 1879.0ms/step | 34.1 samples/s
+Step 6990/10001 | Loss: 3.3444 | LR: 1.06e-04 | 1878.5ms/step | 34.1 samples/s
+Step 7000/10001 | Loss: 3.3163 | LR: 1.05e-04 | 1882.4ms/step | 34.0 samples/s
+Validation at step 7000: Loss = 3.3440, Perplexity = 28.33
+Step 7010/10001 | Loss: 3.2990 | LR: 1.04e-04 | 32043.9ms/step | 2.0 samples/s
+Step 7020/10001 | Loss: 3.3451 | LR: 1.04e-04 | 1880.0ms/step | 34.0 samples/s
+Step 7030/10001 | Loss: 3.2903 | LR: 1.03e-04 | 1883.3ms/step | 34.0 samples/s
+Step 7040/10001 | Loss: 3.3623 | LR: 1.02e-04 | 1883.3ms/step | 34.0 samples/s
+Step 7050/10001 | Loss: 3.3405 | LR: 1.02e-04 | 1879.0ms/step | 34.1 samples/s
+Step 7060/10001 | Loss: 3.3433 | LR: 1.01e-04 | 1878.0ms/step | 34.1 samples/s
+Step 7070/10001 | Loss: 3.3389 | LR: 1.01e-04 | 1882.0ms/step | 34.0 samples/s
+Step 7080/10001 | Loss: 3.3116 | LR: 9.99e-05 | 1882.5ms/step | 34.0 samples/s
+Step 7090/10001 | Loss: 3.3317 | LR: 9.93e-05 | 1883.9ms/step | 34.0 samples/s
+Step 7100/10001 | Loss: 3.3280 | LR: 9.86e-05 | 1881.8ms/step | 34.0 samples/s
+Step 7110/10001 | Loss: 3.3144 | LR: 9.80e-05 | 1878.2ms/step | 34.1 samples/s
+Step 7120/10001 | Loss: 3.3241 | LR: 9.74e-05 | 1929.0ms/step | 33.2 samples/s
+Step 7130/10001 | Loss: 3.3498 | LR: 9.68e-05 | 1881.0ms/step | 34.0 samples/s
+Step 7140/10001 | Loss: 3.3445 | LR: 9.61e-05 | 1880.5ms/step | 34.0 samples/s
+Step 7150/10001 | Loss: 3.3515 | LR: 9.55e-05 | 1879.0ms/step | 34.1 samples/s
+Step 7160/10001 | Loss: 3.3643 | LR: 9.49e-05 | 1881.5ms/step | 34.0 samples/s
+Step 7170/10001 | Loss: 3.3240 | LR: 9.43e-05 | 1880.9ms/step | 34.0 samples/s
+Step 7180/10001 | Loss: 3.3254 | LR: 9.36e-05 | 1880.6ms/step | 34.0 samples/s
+Step 7190/10001 | Loss: 3.3400 | LR: 9.30e-05 | 1883.3ms/step | 34.0 samples/s
+Step 7200/10001 | Loss: 3.3432 | LR: 9.24e-05 | 1882.2ms/step | 34.0 samples/s
+Step 7210/10001 | Loss: 3.3519 | LR: 9.18e-05 | 1880.5ms/step | 34.0 samples/s
+Step 7220/10001 | Loss: 3.3295 | LR: 9.12e-05 | 1883.0ms/step | 34.0 samples/s
+Step 7230/10001 | Loss: 3.3411 | LR: 9.06e-05 | 1883.0ms/step | 34.0 samples/s
+Step 7240/10001 | Loss: 3.3121 | LR: 9.00e-05 | 1883.9ms/step | 34.0 samples/s
+Step 7250/10001 | Loss: 3.3494 | LR: 8.93e-05 | 1880.8ms/step | 34.0 samples/s
+Validation at step 7250: Loss = 3.3389, Perplexity = 28.19
+Step 7260/10001 | Loss: 3.3059 | LR: 8.87e-05 | 32032.7ms/step | 2.0 samples/s
+Step 7270/10001 | Loss: 3.3524 | LR: 8.81e-05 | 1881.0ms/step | 34.0 samples/s
+Step 7280/10001 | Loss: 3.3586 | LR: 8.75e-05 | 1883.5ms/step | 34.0 samples/s
+Step 7290/10001 | Loss: 3.3337 | LR: 8.69e-05 | 1881.7ms/step | 34.0 samples/s
+Step 7300/10001 | Loss: 3.3387 | LR: 8.63e-05 | 1879.8ms/step | 34.0 samples/s
+Step 7310/10001 | Loss: 3.3265 | LR: 8.57e-05 | 1881.0ms/step | 34.0 samples/s
+Step 7320/10001 | Loss: 3.3327 | LR: 8.51e-05 | 1883.7ms/step | 34.0 samples/s
+Step 7330/10001 | Loss: 3.3491 | LR: 8.45e-05 | 1882.6ms/step | 34.0 samples/s
+Step 7340/10001 | Loss: 3.3099 | LR: 8.39e-05 | 1880.1ms/step | 34.0 samples/s
+Step 7350/10001 | Loss: 3.3459 | LR: 8.34e-05 | 1882.0ms/step | 34.0 samples/s
+Step 7360/10001 | Loss: 3.3496 | LR: 8.28e-05 | 1881.0ms/step | 34.0 samples/s
+Step 7370/10001 | Loss: 3.3419 | LR: 8.22e-05 | 1879.2ms/step | 34.1 samples/s
+Step 7380/10001 | Loss: 3.3537 | LR: 8.16e-05 | 1880.9ms/step | 34.0 samples/s
+Step 7390/10001 | Loss: 3.3247 | LR: 8.10e-05 | 1882.8ms/step | 34.0 samples/s
+Step 7400/10001 | Loss: 3.3359 | LR: 8.04e-05 | 1880.5ms/step | 34.0 samples/s
+Step 7410/10001 | Loss: 3.3516 | LR: 7.98e-05 | 1882.7ms/step | 34.0 samples/s
+Step 7420/10001 | Loss: 3.3152 | LR: 7.93e-05 | 1886.4ms/step | 33.9 samples/s
+Step 7430/10001 | Loss: 3.3415 | LR: 7.87e-05 | 1880.4ms/step | 34.0 samples/s
+Step 7440/10001 | Loss: 3.3187 | LR: 7.81e-05 | 1880.5ms/step | 34.0 samples/s
+Step 7450/10001 | Loss: 3.3418 | LR: 7.75e-05 | 1881.6ms/step | 34.0 samples/s
+Step 7460/10001 | Loss: 3.3315 | LR: 7.69e-05 | 1879.8ms/step | 34.0 samples/s
+Step 7470/10001 | Loss: 3.3125 | LR: 7.64e-05 | 1880.6ms/step | 34.0 samples/s
+Step 7480/10001 | Loss: 3.3247 | LR: 7.58e-05 | 1879.6ms/step | 34.0 samples/s
+Step 7490/10001 | Loss: 3.3148 | LR: 7.52e-05 | 1880.3ms/step | 34.0 samples/s
+Step 7500/10001 | Loss: 3.3605 | LR: 7.47e-05 | 1880.5ms/step | 34.0 samples/s
+Validation at step 7500: Loss = 3.3347, Perplexity = 28.07
+Saved checkpoint to checkpoints/run_20260430_020658/checkpoint-7500
+Step 7510/10001 | Loss: 3.3511 | LR: 7.41e-05 | 32482.7ms/step | 2.0 samples/s
+Step 7520/10001 | Loss: 3.3006 | LR: 7.35e-05 | 1880.9ms/step | 34.0 samples/s
+Step 7530/10001 | Loss: 3.3507 | LR: 7.30e-05 | 1879.9ms/step | 34.0 samples/s
+Step 7540/10001 | Loss: 3.3331 | LR: 7.24e-05 | 1878.2ms/step | 34.1 samples/s
+Step 7550/10001 | Loss: 3.3161 | LR: 7.19e-05 | 1879.1ms/step | 34.1 samples/s
+Step 7560/10001 | Loss: 3.3184 | LR: 7.13e-05 | 1877.7ms/step | 34.1 samples/s
+Step 7570/10001 | Loss: 3.3190 | LR: 7.08e-05 | 1881.2ms/step | 34.0 samples/s
+Step 7580/10001 | Loss: 3.3048 | LR: 7.02e-05 | 1879.6ms/step | 34.1 samples/s
+Step 7590/10001 | Loss: 3.3376 | LR: 6.97e-05 | 1881.1ms/step | 34.0 samples/s
+Step 7600/10001 | Loss: 3.3496 | LR: 6.91e-05 | 1926.2ms/step | 33.2 samples/s
+Step 7610/10001 | Loss: 3.3048 | LR: 6.86e-05 | 1879.8ms/step | 34.0 samples/s
+Step 7620/10001 | Loss: 3.3632 | LR: 6.80e-05 | 1879.0ms/step | 34.1 samples/s
+Step 7630/10001 | Loss: 3.3316 | LR: 6.75e-05 | 1881.0ms/step | 34.0 samples/s
+Step 7640/10001 | Loss: 3.3225 | LR: 6.69e-05 | 1876.3ms/step | 34.1 samples/s
+Step 7650/10001 | Loss: 3.3408 | LR: 6.64e-05 | 1879.2ms/step | 34.1 samples/s
+Step 7660/10001 | Loss: 3.3226 | LR: 6.59e-05 | 1880.4ms/step | 34.0 samples/s
+Step 7670/10001 | Loss: 3.3217 | LR: 6.53e-05 | 1876.9ms/step | 34.1 samples/s
+Step 7680/10001 | Loss: 3.3452 | LR: 6.48e-05 | 1879.7ms/step | 34.0 samples/s
+Step 7690/10001 | Loss: 3.3244 | LR: 6.43e-05 | 1881.5ms/step | 34.0 samples/s
+Step 7700/10001 | Loss: 3.3123 | LR: 6.37e-05 | 1878.1ms/step | 34.1 samples/s
+Step 7710/10001 | Loss: 3.3322 | LR: 6.32e-05 | 1878.8ms/step | 34.1 samples/s
+Step 7720/10001 | Loss: 3.3309 | LR: 6.27e-05 | 1879.2ms/step | 34.1 samples/s
+Step 7730/10001 | Loss: 3.3297 | LR: 6.21e-05 | 1881.6ms/step | 34.0 samples/s
+Step 7740/10001 | Loss: 3.3151 | LR: 6.16e-05 | 1879.5ms/step | 34.1 samples/s
+Step 7750/10001 | Loss: 3.3182 | LR: 6.11e-05 | 1879.3ms/step | 34.1 samples/s
+Validation at step 7750: Loss = 3.3317, Perplexity = 27.99
+Step 7760/10001 | Loss: 3.3547 | LR: 6.06e-05 | 32025.9ms/step | 2.0 samples/s
+Step 7770/10001 | Loss: 3.3301 | LR: 6.01e-05 | 1876.5ms/step | 34.1 samples/s
+Step 7780/10001 | Loss: 3.3134 | LR: 5.96e-05 | 1876.6ms/step | 34.1 samples/s
+Step 7790/10001 | Loss: 3.3239 | LR: 5.90e-05 | 1879.4ms/step | 34.1 samples/s
+Step 7800/10001 | Loss: 3.3083 | LR: 5.85e-05 | 1883.7ms/step | 34.0 samples/s
+Step 7810/10001 | Loss: 3.3411 | LR: 5.80e-05 | 1884.0ms/step | 34.0 samples/s
+Step 7820/10001 | Loss: 3.3386 | LR: 5.75e-05 | 1880.0ms/step | 34.0 samples/s
+Step 7830/10001 | Loss: 3.3126 | LR: 5.70e-05 | 1883.7ms/step | 34.0 samples/s
+Step 7840/10001 | Loss: 3.3305 | LR: 5.65e-05 | 1887.3ms/step | 33.9 samples/s
+Step 7850/10001 | Loss: 3.3124 | LR: 5.60e-05 | 1878.9ms/step | 34.1 samples/s
+Step 7860/10001 | Loss: 3.3111 | LR: 5.55e-05 | 1880.5ms/step | 34.0 samples/s
+Step 7870/10001 | Loss: 3.3196 | LR: 5.50e-05 | 1885.0ms/step | 34.0 samples/s
+Step 7880/10001 | Loss: 3.3166 | LR: 5.45e-05 | 1882.3ms/step | 34.0 samples/s
+Step 7890/10001 | Loss: 3.2917 | LR: 5.40e-05 | 1881.3ms/step | 34.0 samples/s
+Step 7900/10001 | Loss: 3.3503 | LR: 5.35e-05 | 1887.6ms/step | 33.9 samples/s
+Step 7910/10001 | Loss: 3.3030 | LR: 5.30e-05 | 1887.2ms/step | 33.9 samples/s
+Step 7920/10001 | Loss: 3.3153 | LR: 5.25e-05 | 1881.5ms/step | 34.0 samples/s
+Step 7930/10001 | Loss: 3.3440 | LR: 5.21e-05 | 1881.8ms/step | 34.0 samples/s
+Step 7940/10001 | Loss: 3.3315 | LR: 5.16e-05 | 1881.8ms/step | 34.0 samples/s
+Step 7950/10001 | Loss: 3.3275 | LR: 5.11e-05 | 1881.3ms/step | 34.0 samples/s
+Step 7960/10001 | Loss: 3.3127 | LR: 5.06e-05 | 1880.0ms/step | 34.0 samples/s
+Step 7970/10001 | Loss: 3.3193 | LR: 5.01e-05 | 1880.7ms/step | 34.0 samples/s
+Step 7980/10001 | Loss: 3.3286 | LR: 4.97e-05 | 1881.4ms/step | 34.0 samples/s
+Step 7990/10001 | Loss: 3.3302 | LR: 4.92e-05 | 1878.1ms/step | 34.1 samples/s
+Step 8000/10001 | Loss: 3.3296 | LR: 4.87e-05 | 1879.3ms/step | 34.1 samples/s
+Validation at step 8000: Loss = 3.3296, Perplexity = 27.93
+Step 8010/10001 | Loss: 3.3151 | LR: 4.83e-05 | 32022.3ms/step | 2.0 samples/s
+Step 8020/10001 | Loss: 3.3212 | LR: 4.78e-05 | 1879.6ms/step | 34.0 samples/s
+Step 8030/10001 | Loss: 3.3305 | LR: 4.73e-05 | 1879.9ms/step | 34.0 samples/s
+Step 8040/10001 | Loss: 3.2963 | LR: 4.69e-05 | 1881.1ms/step | 34.0 samples/s
+Step 8050/10001 | Loss: 3.3375 | LR: 4.64e-05 | 1881.9ms/step | 34.0 samples/s
+Step 8060/10001 | Loss: 3.3208 | LR: 4.59e-05 | 1880.0ms/step | 34.0 samples/s
+Step 8070/10001 | Loss: 3.3271 | LR: 4.55e-05 | 1929.1ms/step | 33.2 samples/s
+Step 8080/10001 | Loss: 3.3155 | LR: 4.50e-05 | 1881.2ms/step | 34.0 samples/s
+Step 8090/10001 | Loss: 3.2927 | LR: 4.46e-05 | 1881.5ms/step | 34.0 samples/s
+Step 8100/10001 | Loss: 3.3454 | LR: 4.41e-05 | 1880.4ms/step | 34.0 samples/s
+Step 8110/10001 | Loss: 3.3151 | LR: 4.37e-05 | 1881.0ms/step | 34.0 samples/s
+Step 8120/10001 | Loss: 3.3487 | LR: 4.32e-05 | 1883.6ms/step | 34.0 samples/s
+Step 8130/10001 | Loss: 3.3121 | LR: 4.28e-05 | 1883.5ms/step | 34.0 samples/s
+Step 8140/10001 | Loss: 3.3180 | LR: 4.23e-05 | 1881.5ms/step | 34.0 samples/s
+Step 8150/10001 | Loss: 3.3241 | LR: 4.19e-05 | 1880.7ms/step | 34.0 samples/s
+Step 8160/10001 | Loss: 3.3549 | LR: 4.15e-05 | 1881.2ms/step | 34.0 samples/s
+Step 8170/10001 | Loss: 3.3282 | LR: 4.10e-05 | 1881.6ms/step | 34.0 samples/s
+Step 8180/10001 | Loss: 3.3219 | LR: 4.06e-05 | 1883.0ms/step | 34.0 samples/s
+Step 8190/10001 | Loss: 3.3488 | LR: 4.02e-05 | 1883.4ms/step | 34.0 samples/s
+Step 8200/10001 | Loss: 3.3338 | LR: 3.97e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8210/10001 | Loss: 3.3221 | LR: 3.93e-05 | 1881.6ms/step | 34.0 samples/s
+Step 8220/10001 | Loss: 3.3172 | LR: 3.89e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8230/10001 | Loss: 3.3135 | LR: 3.84e-05 | 1881.0ms/step | 34.0 samples/s
+Step 8240/10001 | Loss: 3.3210 | LR: 3.80e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8250/10001 | Loss: 3.3343 | LR: 3.76e-05 | 1881.9ms/step | 34.0 samples/s
+Validation at step 8250: Loss = 3.3282, Perplexity = 27.89
+Step 8260/10001 | Loss: 3.2939 | LR: 3.72e-05 | 32048.5ms/step | 2.0 samples/s
+Step 8270/10001 | Loss: 3.3034 | LR: 3.68e-05 | 1881.4ms/step | 34.0 samples/s
+Step 8280/10001 | Loss: 3.3294 | LR: 3.64e-05 | 1881.6ms/step | 34.0 samples/s
+Step 8290/10001 | Loss: 3.2963 | LR: 3.59e-05 | 1882.9ms/step | 34.0 samples/s
+Step 8300/10001 | Loss: 3.3420 | LR: 3.55e-05 | 1880.9ms/step | 34.0 samples/s
+Step 8310/10001 | Loss: 3.3046 | LR: 3.51e-05 | 1881.3ms/step | 34.0 samples/s
+Step 8320/10001 | Loss: 3.3274 | LR: 3.47e-05 | 1882.7ms/step | 34.0 samples/s
+Step 8330/10001 | Loss: 3.3213 | LR: 3.43e-05 | 1881.9ms/step | 34.0 samples/s
+Step 8340/10001 | Loss: 3.3434 | LR: 3.39e-05 | 1882.1ms/step | 34.0 samples/s
+Step 8350/10001 | Loss: 3.3430 | LR: 3.35e-05 | 1884.0ms/step | 34.0 samples/s
+Step 8360/10001 | Loss: 3.3435 | LR: 3.31e-05 | 1880.0ms/step | 34.0 samples/s
+Step 8370/10001 | Loss: 3.3298 | LR: 3.27e-05 | 1880.6ms/step | 34.0 samples/s
+Step 8380/10001 | Loss: 3.3117 | LR: 3.23e-05 | 1883.3ms/step | 34.0 samples/s
+Step 8390/10001 | Loss: 3.3275 | LR: 3.20e-05 | 1880.7ms/step | 34.0 samples/s
+Step 8400/10001 | Loss: 3.3270 | LR: 3.16e-05 | 1880.9ms/step | 34.0 samples/s
+Step 8410/10001 | Loss: 3.3481 | LR: 3.12e-05 | 1881.0ms/step | 34.0 samples/s
+Step 8420/10001 | Loss: 3.2882 | LR: 3.08e-05 | 1882.5ms/step | 34.0 samples/s
+Step 8430/10001 | Loss: 3.3390 | LR: 3.04e-05 | 1882.9ms/step | 34.0 samples/s
+Step 8440/10001 | Loss: 3.3304 | LR: 3.00e-05 | 1883.0ms/step | 34.0 samples/s
+Step 8450/10001 | Loss: 3.3478 | LR: 2.97e-05 | 1881.6ms/step | 34.0 samples/s
+Step 8460/10001 | Loss: 3.3419 | LR: 2.93e-05 | 1882.1ms/step | 34.0 samples/s
+Step 8470/10001 | Loss: 3.3233 | LR: 2.89e-05 | 1881.8ms/step | 34.0 samples/s
+Step 8480/10001 | Loss: 3.3305 | LR: 2.86e-05 | 1882.1ms/step | 34.0 samples/s
+Step 8490/10001 | Loss: 3.3289 | LR: 2.82e-05 | 1881.1ms/step | 34.0 samples/s
+Step 8500/10001 | Loss: 3.3485 | LR: 2.78e-05 | 1880.2ms/step | 34.0 samples/s
+Validation at step 8500: Loss = 3.3273, Perplexity = 27.86
+Step 8510/10001 | Loss: 3.3362 | LR: 2.75e-05 | 32060.2ms/step | 2.0 samples/s
+Step 8520/10001 | Loss: 3.3388 | LR: 2.71e-05 | 1882.3ms/step | 34.0 samples/s
+Step 8530/10001 | Loss: 3.3390 | LR: 2.67e-05 | 1881.8ms/step | 34.0 samples/s
+Step 8540/10001 | Loss: 3.3043 | LR: 2.64e-05 | 1882.3ms/step | 34.0 samples/s
+Step 8550/10001 | Loss: 3.3316 | LR: 2.60e-05 | 1930.5ms/step | 33.2 samples/s
+Step 8560/10001 | Loss: 3.3146 | LR: 2.57e-05 | 1881.5ms/step | 34.0 samples/s
+Step 8570/10001 | Loss: 3.3169 | LR: 2.53e-05 | 1884.1ms/step | 34.0 samples/s
+Step 8580/10001 | Loss: 3.2938 | LR: 2.50e-05 | 1884.4ms/step | 34.0 samples/s
+Step 8590/10001 | Loss: 3.3203 | LR: 2.46e-05 | 1882.7ms/step | 34.0 samples/s
+Step 8600/10001 | Loss: 3.3176 | LR: 2.43e-05 | 1883.1ms/step | 34.0 samples/s
+Step 8610/10001 | Loss: 3.3456 | LR: 2.40e-05 | 1883.6ms/step | 34.0 samples/s
+Step 8620/10001 | Loss: 3.3190 | LR: 2.36e-05 | 1883.4ms/step | 34.0 samples/s
+Step 8630/10001 | Loss: 3.3152 | LR: 2.33e-05 | 1883.7ms/step | 34.0 samples/s
+Step 8640/10001 | Loss: 3.3291 | LR: 2.30e-05 | 1880.9ms/step | 34.0 samples/s
+Step 8650/10001 | Loss: 3.3304 | LR: 2.26e-05 | 1881.4ms/step | 34.0 samples/s
+Step 8660/10001 | Loss: 3.3410 | LR: 2.23e-05 | 1881.1ms/step | 34.0 samples/s
+Step 8670/10001 | Loss: 3.3081 | LR: 2.20e-05 | 1881.6ms/step | 34.0 samples/s
+Step 8680/10001 | Loss: 3.3299 | LR: 2.16e-05 | 1881.9ms/step | 34.0 samples/s
+Step 8690/10001 | Loss: 3.3507 | LR: 2.13e-05 | 1881.4ms/step | 34.0 samples/s
+Step 8700/10001 | Loss: 3.2943 | LR: 2.10e-05 | 1882.7ms/step | 34.0 samples/s
+Step 8710/10001 | Loss: 3.3110 | LR: 2.07e-05 | 1883.4ms/step | 34.0 samples/s
+Step 8720/10001 | Loss: 3.3253 | LR: 2.04e-05 | 1881.9ms/step | 34.0 samples/s
+Step 8730/10001 | Loss: 3.3312 | LR: 2.01e-05 | 1880.7ms/step | 34.0 samples/s
+Step 8740/10001 | Loss: 3.3184 | LR: 1.97e-05 | 1881.0ms/step | 34.0 samples/s
+Step 8750/10001 | Loss: 3.3222 | LR: 1.94e-05 | 1882.5ms/step | 34.0 samples/s
+Validation at step 8750: Loss = 3.3268, Perplexity = 27.85
+Step 8760/10001 | Loss: 3.3385 | LR: 1.91e-05 | 32066.5ms/step | 2.0 samples/s
+Step 8770/10001 | Loss: 3.3451 | LR: 1.88e-05 | 1881.4ms/step | 34.0 samples/s
+Step 8780/10001 | Loss: 3.3262 | LR: 1.85e-05 | 1882.6ms/step | 34.0 samples/s
+Step 8790/10001 | Loss: 3.3407 | LR: 1.82e-05 | 1882.9ms/step | 34.0 samples/s
+Step 8800/10001 | Loss: 3.3387 | LR: 1.79e-05 | 1883.1ms/step | 34.0 samples/s
+Step 8810/10001 | Loss: 3.3099 | LR: 1.76e-05 | 1881.7ms/step | 34.0 samples/s
+Step 8820/10001 | Loss: 3.3136 | LR: 1.73e-05 | 1881.7ms/step | 34.0 samples/s
+Step 8830/10001 | Loss: 3.3190 | LR: 1.71e-05 | 1883.9ms/step | 34.0 samples/s
+Step 8840/10001 | Loss: 3.3499 | LR: 1.68e-05 | 1882.7ms/step | 34.0 samples/s
+Step 8850/10001 | Loss: 3.3346 | LR: 1.65e-05 | 1883.0ms/step | 34.0 samples/s
+Step 8860/10001 | Loss: 3.3113 | LR: 1.62e-05 | 1882.4ms/step | 34.0 samples/s
+Step 8870/10001 | Loss: 3.3080 | LR: 1.59e-05 | 1883.6ms/step | 34.0 samples/s
+Step 8880/10001 | Loss: 3.3148 | LR: 1.56e-05 | 1885.5ms/step | 33.9 samples/s
+Step 8890/10001 | Loss: 3.3349 | LR: 1.54e-05 | 1884.4ms/step | 34.0 samples/s
+Step 8900/10001 | Loss: 3.3120 | LR: 1.51e-05 | 1883.5ms/step | 34.0 samples/s
+Step 8910/10001 | Loss: 3.3282 | LR: 1.48e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8920/10001 | Loss: 3.3192 | LR: 1.46e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8930/10001 | Loss: 3.3284 | LR: 1.43e-05 | 1883.0ms/step | 34.0 samples/s
+Step 8940/10001 | Loss: 3.3042 | LR: 1.40e-05 | 1881.9ms/step | 34.0 samples/s
+Step 8950/10001 | Loss: 3.3231 | LR: 1.38e-05 | 1882.6ms/step | 34.0 samples/s
+Step 8960/10001 | Loss: 3.3184 | LR: 1.35e-05 | 1881.5ms/step | 34.0 samples/s
+Step 8970/10001 | Loss: 3.3225 | LR: 1.33e-05 | 1882.0ms/step | 34.0 samples/s
+Step 8980/10001 | Loss: 3.3131 | LR: 1.30e-05 | 1883.8ms/step | 34.0 samples/s
+Step 8990/10001 | Loss: 3.3457 | LR: 1.28e-05 | 1882.6ms/step | 34.0 samples/s
+Step 9000/10001 | Loss: 3.3443 | LR: 1.25e-05 | 1882.0ms/step | 34.0 samples/s
+Validation at step 9000: Loss = 3.3266, Perplexity = 27.84
+Step 9010/10001 | Loss: 3.3156 | LR: 1.23e-05 | 32069.7ms/step | 2.0 samples/s
+Step 9020/10001 | Loss: 3.3457 | LR: 1.20e-05 | 1933.3ms/step | 33.1 samples/s
+Step 9030/10001 | Loss: 3.3372 | LR: 1.18e-05 | 1881.9ms/step | 34.0 samples/s
+Step 9040/10001 | Loss: 3.3173 | LR: 1.15e-05 | 1882.4ms/step | 34.0 samples/s
+Step 9050/10001 | Loss: 3.3276 | LR: 1.13e-05 | 1883.8ms/step | 34.0 samples/s
+Step 9060/10001 | Loss: 3.3270 | LR: 1.11e-05 | 1884.3ms/step | 34.0 samples/s
+Step 9070/10001 | Loss: 3.3309 | LR: 1.08e-05 | 1881.9ms/step | 34.0 samples/s
+Step 9080/10001 | Loss: 3.3383 | LR: 1.06e-05 | 1882.1ms/step | 34.0 samples/s
+Step 9090/10001 | Loss: 3.3199 | LR: 1.04e-05 | 1880.5ms/step | 34.0 samples/s
+Step 9100/10001 | Loss: 3.3209 | LR: 1.01e-05 | 1880.4ms/step | 34.0 samples/s
+Step 9110/10001 | Loss: 3.3290 | LR: 9.92e-06 | 1880.5ms/step | 34.0 samples/s
+Step 9120/10001 | Loss: 3.3120 | LR: 9.70e-06 | 1881.6ms/step | 34.0 samples/s
+Step 9130/10001 | Loss: 3.3232 | LR: 9.49e-06 | 1880.4ms/step | 34.0 samples/s
+Step 9140/10001 | Loss: 3.3358 | LR: 9.27e-06 | 1880.3ms/step | 34.0 samples/s
+Step 9150/10001 | Loss: 3.2918 | LR: 9.06e-06 | 1879.6ms/step | 34.0 samples/s
+Step 9160/10001 | Loss: 3.3040 | LR: 8.85e-06 | 1881.1ms/step | 34.0 samples/s
+Step 9170/10001 | Loss: 3.3094 | LR: 8.64e-06 | 1880.0ms/step | 34.0 samples/s
+Step 9180/10001 | Loss: 3.3155 | LR: 8.43e-06 | 1882.4ms/step | 34.0 samples/s
+Step 9190/10001 | Loss: 3.3355 | LR: 8.23e-06 | 1882.5ms/step | 34.0 samples/s
+Step 9200/10001 | Loss: 3.3033 | LR: 8.03e-06 | 1881.1ms/step | 34.0 samples/s
+Step 9210/10001 | Loss: 3.3359 | LR: 7.83e-06 | 1884.1ms/step | 34.0 samples/s
+Step 9220/10001 | Loss: 3.3552 | LR: 7.64e-06 | 1881.8ms/step | 34.0 samples/s
+Step 9230/10001 | Loss: 3.3077 | LR: 7.44e-06 | 1883.3ms/step | 34.0 samples/s
+Step 9240/10001 | Loss: 3.3352 | LR: 7.25e-06 | 1882.2ms/step | 34.0 samples/s
+Step 9250/10001 | Loss: 3.3396 | LR: 7.06e-06 | 1884.0ms/step | 34.0 samples/s
+Validation at step 9250: Loss = 3.3265, Perplexity = 27.84
+Step 9260/10001 | Loss: 3.3441 | LR: 6.88e-06 | 32072.3ms/step | 2.0 samples/s
+Step 9270/10001 | Loss: 3.3168 | LR: 6.69e-06 | 1883.1ms/step | 34.0 samples/s
+Step 9280/10001 | Loss: 3.3410 | LR: 6.51e-06 | 1882.6ms/step | 34.0 samples/s
+Step 9290/10001 | Loss: 3.3157 | LR: 6.34e-06 | 1883.0ms/step | 34.0 samples/s
+Step 9300/10001 | Loss: 3.3411 | LR: 6.16e-06 | 1883.4ms/step | 34.0 samples/s
+Step 9310/10001 | Loss: 3.2889 | LR: 5.99e-06 | 1884.1ms/step | 34.0 samples/s
+Step 9320/10001 | Loss: 3.2743 | LR: 5.81e-06 | 1882.4ms/step | 34.0 samples/s
+Step 9330/10001 | Loss: 3.3277 | LR: 5.64e-06 | 1883.0ms/step | 34.0 samples/s
+Step 9340/10001 | Loss: 3.3088 | LR: 5.48e-06 | 1883.8ms/step | 34.0 samples/s
+Step 9350/10001 | Loss: 3.3461 | LR: 5.31e-06 | 1882.7ms/step | 34.0 samples/s
+Step 9360/10001 | Loss: 3.3287 | LR: 5.15e-06 | 1881.6ms/step | 34.0 samples/s
+Step 9370/10001 | Loss: 3.3659 | LR: 4.99e-06 | 1884.8ms/step | 34.0 samples/s
+Step 9380/10001 | Loss: 3.3261 | LR: 4.84e-06 | 1886.4ms/step | 33.9 samples/s
+Step 9390/10001 | Loss: 3.3432 | LR: 4.68e-06 | 1882.6ms/step | 34.0 samples/s
+Step 9400/10001 | Loss: 3.3190 | LR: 4.53e-06 | 1882.5ms/step | 34.0 samples/s
+Step 9410/10001 | Loss: 3.3544 | LR: 4.38e-06 | 1882.0ms/step | 34.0 samples/s
+Step 9420/10001 | Loss: 3.3112 | LR: 4.24e-06 | 1883.1ms/step | 34.0 samples/s
+Step 9430/10001 | Loss: 3.3244 | LR: 4.09e-06 | 1885.1ms/step | 34.0 samples/s
+Step 9440/10001 | Loss: 3.3281 | LR: 3.95e-06 | 1884.9ms/step | 34.0 samples/s
+Step 9450/10001 | Loss: 3.3199 | LR: 3.81e-06 | 1884.2ms/step | 34.0 samples/s
+Step 9460/10001 | Loss: 3.3501 | LR: 3.67e-06 | 1885.6ms/step | 33.9 samples/s
+Step 9470/10001 | Loss: 3.3029 | LR: 3.54e-06 | 1883.9ms/step | 34.0 samples/s
+Step 9480/10001 | Loss: 3.3281 | LR: 3.41e-06 | 1883.7ms/step | 34.0 samples/s
+Step 9490/10001 | Loss: 3.3442 | LR: 3.28e-06 | 1886.7ms/step | 33.9 samples/s
+Step 9500/10001 | Loss: 3.3323 | LR: 3.15e-06 | 1938.1ms/step | 33.0 samples/s
+Validation at step 9500: Loss = 3.3264, Perplexity = 27.84
+Step 9510/10001 | Loss: 3.3407 | LR: 3.03e-06 | 32098.5ms/step | 2.0 samples/s
+Step 9520/10001 | Loss: 3.3194 | LR: 2.91e-06 | 1883.5ms/step | 34.0 samples/s
+Step 9530/10001 | Loss: 3.3119 | LR: 2.79e-06 | 1882.4ms/step | 34.0 samples/s
+Step 9540/10001 | Loss: 3.3254 | LR: 2.67e-06 | 1882.9ms/step | 34.0 samples/s
+Step 9550/10001 | Loss: 3.3601 | LR: 2.56e-06 | 1886.6ms/step | 33.9 samples/s
+Step 9560/10001 | Loss: 3.3164 | LR: 2.44e-06 | 1884.1ms/step | 34.0 samples/s
+Step 9570/10001 | Loss: 3.3203 | LR: 2.33e-06 | 1884.6ms/step | 34.0 samples/s
+Step 9580/10001 | Loss: 3.3140 | LR: 2.23e-06 | 1884.8ms/step | 34.0 samples/s
+Step 9590/10001 | Loss: 3.3421 | LR: 2.12e-06 | 1885.6ms/step | 33.9 samples/s
+Step 9600/10001 | Loss: 3.3284 | LR: 2.02e-06 | 1885.5ms/step | 33.9 samples/s
+Step 9610/10001 | Loss: 3.3044 | LR: 1.92e-06 | 1886.4ms/step | 33.9 samples/s
+Step 9620/10001 | Loss: 3.3191 | LR: 1.82e-06 | 1888.7ms/step | 33.9 samples/s
+Step 9630/10001 | Loss: 3.3440 | LR: 1.73e-06 | 1885.3ms/step | 33.9 samples/s
+Step 9640/10001 | Loss: 3.3393 | LR: 1.64e-06 | 1884.5ms/step | 34.0 samples/s
+Step 9650/10001 | Loss: 3.3325 | LR: 1.55e-06 | 1888.9ms/step | 33.9 samples/s
+Step 9660/10001 | Loss: 3.3148 | LR: 1.46e-06 | 1888.9ms/step | 33.9 samples/s
+Step 9670/10001 | Loss: 3.2976 | LR: 1.38e-06 | 1886.8ms/step | 33.9 samples/s
+Step 9680/10001 | Loss: 3.3045 | LR: 1.30e-06 | 1886.6ms/step | 33.9 samples/s
+Step 9690/10001 | Loss: 3.3334 | LR: 1.22e-06 | 1887.4ms/step | 33.9 samples/s
+Step 9700/10001 | Loss: 3.3266 | LR: 1.14e-06 | 1887.7ms/step | 33.9 samples/s
+Step 9710/10001 | Loss: 3.3309 | LR: 1.06e-06 | 1887.0ms/step | 33.9 samples/s
+Step 9720/10001 | Loss: 3.3355 | LR: 9.93e-07 | 1888.3ms/step | 33.9 samples/s
+Step 9730/10001 | Loss: 3.3240 | LR: 9.24e-07 | 1883.7ms/step | 34.0 samples/s
+Step 9740/10001 | Loss: 3.3123 | LR: 8.57e-07 | 1883.0ms/step | 34.0 samples/s
+Step 9750/10001 | Loss: 3.3396 | LR: 7.92e-07 | 1887.7ms/step | 33.9 samples/s
+Validation at step 9750: Loss = 3.3264, Perplexity = 27.84
+Step 9760/10001 | Loss: 3.3191 | LR: 7.31e-07 | 32092.3ms/step | 2.0 samples/s
+Saved checkpoint to checkpoints/run_20260430_020658/checkpoint-9764

epfl_final/wandb/run-20260430_020702-cxasagq2/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.1.post1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+GitPython==3.1.49
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.0
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.11.0
+huggingface_hub==1.12.2
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260430_020702-cxasagq2/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-04-30T02:07:02.327832Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-workspace-4-0-6",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "4381286359040"
+    }
+  },
+  "memory":  {
+    "total":  "1623005847552"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-d0294574-a180-e984-6ca3-0573e9724f2e"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "ocqykme6tsi3am7h67amdeb6fgb1wopc"
+}

epfl_final/wandb/run-20260430_020702-cxasagq2/files/wandb-summary.json

@@ -0,0 +1 @@
+{"speed/ms_per_step":32092.307949066162,"eval/loss":3.32643155258894,"_step":9760,"_timestamp":1.777545066290692e+09,"eval/perplexity":27.83882286958296,"train/loss":3.319102776050568,"speed/samples_per_sec":1.994247347419658,"train/learning_rate":7.305906022035125e-07,"_runtime":30255.948872206,"train/epoch":0,"speed/steps_per_sec":0.031160114803432157,"train/global_step":9760,"_wandb":{"runtime":30255}}

epfl_final/wandb/run-20260430_020702-cxasagq2/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-30 02:07:02,345 INFO    MainThread:283 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-04-30 02:07:02,345 INFO    MainThread:283 [wandb_setup.py:_flush():81] Configure stats pid to 283
+2026-04-30 02:07:02,345 INFO    MainThread:283 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 02:07:02,345 INFO    MainThread:283 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_020702-cxasagq2/logs/debug.log
+2026-04-30 02:07:02,346 INFO    MainThread:283 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_020702-cxasagq2/logs/debug-internal.log
+2026-04-30 02:07:02,346 INFO    MainThread:283 [wandb_init.py:init():850] calling init triggers
+2026-04-30 02:07:02,346 INFO    MainThread:283 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 2048, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_020658', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10001, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'flash_attention': True, 'attention_type': 'dense', 'sparse_block_size': 128, 'sparse_num_random_blocks': 1, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, '_wandb': {}}
+2026-04-30 02:07:02,346 INFO    MainThread:283 [wandb_init.py:init():898] starting backend
+2026-04-30 02:07:02,579 INFO    MainThread:283 [wandb_init.py:init():913] sending inform_init request
+2026-04-30 02:07:02,907 INFO    MainThread:283 [wandb_init.py:init():918] backend started and connected
+2026-04-30 02:07:02,911 INFO    MainThread:283 [wandb_init.py:init():988] updated telemetry
+2026-04-30 02:07:02,912 INFO    MainThread:283 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-04-30 02:07:03,542 INFO    MainThread:283 [wandb_init.py:init():1056] starting run threads in backend
+2026-04-30 02:07:03,629 INFO    MainThread:283 [wandb_run.py:_console_start():2554] atexit reg
+2026-04-30 02:07:03,629 INFO    MainThread:283 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-04-30 02:07:03,629 INFO    MainThread:283 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-04-30 02:07:03,629 INFO    MainThread:283 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-04-30 02:07:03,632 INFO    MainThread:283 [wandb_init.py:init():1094] run started, returning control to user process
+2026-04-30 10:31:19,490 INFO    MainThread:283 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/cxasagq2
+2026-04-30 10:31:19,491 INFO    MainThread:283 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-04-30 10:31:19,492 INFO    MainThread:283 [wandb_run.py:_restore():2502] restore
+2026-04-30 10:31:19,492 INFO    MainThread:283 [wandb_run.py:_restore():2508] restore done
+2026-04-30 10:31:21,031 INFO    MainThread:283 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/wandb/run-20260430_020702-cxasagq2/run-cxasagq2.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:504d8cfb41c9429a5f833188897512bfce48452b458ea89dd377e22894e654e1
+size 4898662

epfl_final/wandb/run-20260430_021108-ngpqwy44/files/config.yaml

@@ -0,0 +1,131 @@
+_wandb:
+    value:
+        cli_version: 0.26.1
+        e:
+            7fambcx7u8b13zjiikjiufanttbl1jhf:
+                codePath: SparseAtt/train.py
+                codePathLocal: SparseAtt/train.py
+                cpu_count: 96
+                cpu_count_logical: 192
+                cudaVersion: "13.0"
+                disk:
+                    /:
+                        total: "9371709865984"
+                        used: "4381359190016"
+                email: ruben.navasardyan@mail.utoronto.ca
+                executable: /usr/bin/python3
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7
+                host: h100-workspace-5-0-3
+                memory:
+                    total: "1623005847552"
+                os: Linux-6.8.0-59-generic-x86_64-with-glibc2.35
+                program: /mloscratch/homes/navasard/sparse/SparseAtt/train.py
+                python: CPython 3.10.12
+                root: /mloscratch/homes/navasard/sparse
+                startedAt: "2026-04-30T02:11:08.532189Z"
+                writerId: 7fambcx7u8b13zjiikjiufanttbl1jhf
+        m: []
+        python_version: 3.10.12
+        t:
+            "1":
+                - 1
+                - 49
+                - 51
+            "2":
+                - 1
+                - 11
+                - 49
+                - 51
+                - 71
+            "3":
+                - 2
+                - 15
+                - 16
+                - 61
+            "4": 3.10.12
+            "5": 0.26.1
+            "12": 0.26.1
+            "13": linux-x86_64
+attention_type:
+    value: sparse_stochastic
+checkpoint_activations:
+    value: false
+eval_batches:
+    value: 500
+eval_steps:
+    value: 250
+flash_attention:
+    value: true
+gradient_accumulation_steps:
+    value: 8
+init_from_scratch:
+    value: true
+learning_rate:
+    value: 0.0005
+logging_steps:
+    value: 10
+lr_scheduler:
+    value: cosine
+max_grad_norm:
+    value: 1
+max_seq_length:
+    value: 2048
+max_steps:
+    value: 10001
+mixed_precision:
+    value: bf16
+model_dtype:
+    value: bfloat16
+model_path:
+    value: /mloscratch/homes/navasard/qwen3_06b_base
+num_epochs:
+    value: 1
+num_workers:
+    value: 8
+output_dir:
+    value: ./checkpoints/run_20260430_021105
+per_device_batch_size:
+    value: 8
+resume_from_checkpoint:
+    value: null
+save_steps:
+    value: 2500
+seed:
+    value: 42
+sparse_block_size:
+    value: 128
+sparse_mask_pool_size:
+    value: 1000
+sparse_num_random_blocks:
+    value: 7
+sparse_val_dense_compare:
+    value: false
+strategy:
+    value: ddp
+train_path:
+    value: /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized
+val_path:
+    value: /mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized
+verbose:
+    value: true
+wandb_enabled:
+    value: true
+wandb_name:
+    value: null
+wandb_project:
+    value: stochastic_sparsity
+wandb_tags:
+    value:
+        - qwen3
+        - 0.6b
+warmup_steps:
+    value: 100
+weight_decay:
+    value: 0.01

epfl_final/wandb/run-20260430_021108-ngpqwy44/files/output.log

@@ -0,0 +1,1122 @@
+Loading tokenizer from /mloscratch/homes/navasard/qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 1024
+  num_hidden_layers: 28
+  num_attention_heads: 16
+  num_key_value_heads: 8
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000
+  attention_type: sparse_stochastic
+  sparse_block_size: 128
+  sparse_num_random_blocks: 7
+  sparse_mask_pool_size: 1000
+  sparse_max_seq_length: 2048
+
+Parameter count:
+  Total parameters: 596,049,920
+  Trainable parameters: 596,049,920
+  Non-trainable parameters: 0
+  Model size (BF16): 1.19 GB
+
+--------------------------------------------------------------------------------
+LAYER-BY-LAYER SUMMARY
+--------------------------------------------------------------------------------
+/home/navasard/.local/lib/python3.10/site-packages/torch/nn/attention/flex_attention.py:1704: UserWarning: flex_attention called without torch.compile() - this will use an unfused implementation that materializes the full scores matrix instead of generating a fused kernel.
+
+SOLUTION: Use torch.compile(flex_attention)(...)
+
+If you want to debug your score_mod/mask_mod, you can set:
+torch.nn.attention.flex_attention._FLEX_ATTENTION_DISABLE_COMPILE_DEBUG = True
+
+This will allow you to use print statements or breakpoints. Note: This doesn't work with the backwards pass and may produce incorrect results.
+  _warn_once(
+================================================================================================================================================================
+Layer (type:depth-idx)                                       Input Shape               Output Shape              Param #                   Trainable
+================================================================================================================================================================
+Qwen3ForCausalLM                                             [8, 2048]                 --                        --                        True
+├─Qwen3Model: 1-1                                            --                        [8, 2048, 1024]           --                        True
+│    └─Embedding: 2-1                                        [8, 2048]                 [8, 2048, 1024]           155,582,464               True
+│    └─ModuleList: 2-2                                       --                        --                        --                        True
+│    │    └─Qwen3DecoderLayer: 3-1                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-2                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-3                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-4                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-5                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-6                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-7                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-8                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-9                           [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-10                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-11                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-12                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-13                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-14                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-15                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-16                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-17                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-18                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-19                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-20                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-21                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-22                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-23                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-24                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-25                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-26                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-27                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    │    └─Qwen3DecoderLayer: 3-28                          [8, 2048, 1024]           [8, 2048, 1024]           15,730,944                True
+│    └─Qwen3RMSNorm: 2-3                                     [8, 2048, 1024]           [8, 2048, 1024]           1,024                     True
+├─Linear: 1-2                                                [8, 2048, 1024]           [8, 2048, 151936]         155,582,464               True
+================================================================================================================================================================
+Total params: 751,632,384
+Trainable params: 751,632,384
+Non-trainable params: 0
+Total mult-adds (G): 6.01
+================================================================================================================================================================
+Input size (MB): 0.13
+Forward/backward pass size (MB): 42035.31
+Params size (MB): 1503.26
+Estimated Total Size (MB): 43538.71
+================================================================================================================================================================
+================================================================================
+
+Skipping flash-attention hook for sparse_stochastic mode
+Loading dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized...
+Loaded 781148 pre-tokenized examples
+Dataset split: 624918 train, 156230 validation
+Training examples: 624918
+Validation examples: 156230
+Steps per epoch: 9764
+Total training steps: 10001
+Starting training...
+Step 10/10001 | Loss: 11.9163 | LR: 5.00e-05 | 7610.5ms/step | 8.4 samples/s
+Step 20/10001 | Loss: 10.6687 | LR: 1.00e-04 | 1385.2ms/step | 46.2 samples/s
+Step 30/10001 | Loss: 9.7184 | LR: 1.50e-04 | 1384.9ms/step | 46.2 samples/s
+Step 40/10001 | Loss: 8.5675 | LR: 2.00e-04 | 1388.2ms/step | 46.1 samples/s
+Step 50/10001 | Loss: 8.0248 | LR: 2.50e-04 | 1391.5ms/step | 46.0 samples/s
+Step 60/10001 | Loss: 7.9159 | LR: 3.00e-04 | 1396.6ms/step | 45.8 samples/s
+Step 70/10001 | Loss: 7.6968 | LR: 3.50e-04 | 1405.2ms/step | 45.5 samples/s
+Step 80/10001 | Loss: 7.4901 | LR: 4.00e-04 | 1410.0ms/step | 45.4 samples/s
+Step 90/10001 | Loss: 7.3149 | LR: 4.50e-04 | 1411.4ms/step | 45.3 samples/s
+Step 100/10001 | Loss: 7.1345 | LR: 5.00e-04 | 1408.4ms/step | 45.4 samples/s
+Step 110/10001 | Loss: 7.0267 | LR: 5.00e-04 | 1411.4ms/step | 45.3 samples/s
+Step 120/10001 | Loss: 6.8715 | LR: 5.00e-04 | 1412.8ms/step | 45.3 samples/s
+Step 130/10001 | Loss: 6.8415 | LR: 5.00e-04 | 1410.9ms/step | 45.4 samples/s
+Step 140/10001 | Loss: 6.6980 | LR: 5.00e-04 | 1412.8ms/step | 45.3 samples/s
+Step 150/10001 | Loss: 6.6183 | LR: 5.00e-04 | 1411.5ms/step | 45.3 samples/s
+Step 160/10001 | Loss: 6.6109 | LR: 5.00e-04 | 1409.0ms/step | 45.4 samples/s
+Step 170/10001 | Loss: 6.4796 | LR: 5.00e-04 | 1409.9ms/step | 45.4 samples/s
+Step 180/10001 | Loss: 6.4388 | LR: 5.00e-04 | 1408.2ms/step | 45.4 samples/s
+Step 190/10001 | Loss: 6.3359 | LR: 5.00e-04 | 1410.3ms/step | 45.4 samples/s
+Step 200/10001 | Loss: 6.3463 | LR: 5.00e-04 | 1417.7ms/step | 45.1 samples/s
+Step 210/10001 | Loss: 6.2423 | LR: 5.00e-04 | 1415.8ms/step | 45.2 samples/s
+Step 220/10001 | Loss: 6.2009 | LR: 5.00e-04 | 1416.0ms/step | 45.2 samples/s
+Step 230/10001 | Loss: 6.1432 | LR: 5.00e-04 | 1416.8ms/step | 45.2 samples/s
+Step 240/10001 | Loss: 6.0981 | LR: 5.00e-04 | 1419.4ms/step | 45.1 samples/s
+Step 250/10001 | Loss: 6.0170 | LR: 5.00e-04 | 1416.1ms/step | 45.2 samples/s
+Validation at step 250: Loss = 5.9934, Perplexity = 400.76
+Step 260/10001 | Loss: 5.9828 | LR: 5.00e-04 | 30292.5ms/step | 2.1 samples/s
+Step 270/10001 | Loss: 5.9230 | LR: 5.00e-04 | 1420.0ms/step | 45.1 samples/s
+Step 280/10001 | Loss: 5.8514 | LR: 5.00e-04 | 1420.3ms/step | 45.1 samples/s
+Step 290/10001 | Loss: 5.8398 | LR: 5.00e-04 | 1423.7ms/step | 45.0 samples/s
+Step 300/10001 | Loss: 5.7919 | LR: 4.99e-04 | 1420.0ms/step | 45.1 samples/s
+Step 310/10001 | Loss: 5.7385 | LR: 4.99e-04 | 1416.0ms/step | 45.2 samples/s
+Step 320/10001 | Loss: 5.6764 | LR: 4.99e-04 | 1417.7ms/step | 45.1 samples/s
+Step 330/10001 | Loss: 5.6884 | LR: 4.99e-04 | 1421.0ms/step | 45.0 samples/s
+Step 340/10001 | Loss: 5.6007 | LR: 4.99e-04 | 1423.2ms/step | 45.0 samples/s
+Step 350/10001 | Loss: 5.6036 | LR: 4.99e-04 | 1421.8ms/step | 45.0 samples/s
+Step 360/10001 | Loss: 5.5107 | LR: 4.99e-04 | 1420.1ms/step | 45.1 samples/s
+Step 370/10001 | Loss: 5.4832 | LR: 4.99e-04 | 1415.9ms/step | 45.2 samples/s
+Step 380/10001 | Loss: 5.4519 | LR: 4.99e-04 | 1415.6ms/step | 45.2 samples/s
+Step 390/10001 | Loss: 5.4098 | LR: 4.99e-04 | 1416.0ms/step | 45.2 samples/s
+Step 400/10001 | Loss: 5.3818 | LR: 4.99e-04 | 1414.2ms/step | 45.3 samples/s
+Step 410/10001 | Loss: 5.3202 | LR: 4.99e-04 | 1410.7ms/step | 45.4 samples/s
+Step 420/10001 | Loss: 5.2744 | LR: 4.99e-04 | 1414.7ms/step | 45.2 samples/s
+Step 430/10001 | Loss: 5.2484 | LR: 4.99e-04 | 1418.4ms/step | 45.1 samples/s
+Step 440/10001 | Loss: 5.2214 | LR: 4.99e-04 | 1418.2ms/step | 45.1 samples/s
+Step 450/10001 | Loss: 5.1965 | LR: 4.98e-04 | 1417.5ms/step | 45.2 samples/s
+Step 460/10001 | Loss: 5.1433 | LR: 4.98e-04 | 1414.6ms/step | 45.2 samples/s
+Step 470/10001 | Loss: 5.1334 | LR: 4.98e-04 | 1417.7ms/step | 45.1 samples/s
+Step 480/10001 | Loss: 5.0865 | LR: 4.98e-04 | 1420.4ms/step | 45.1 samples/s
+Step 490/10001 | Loss: 5.0823 | LR: 4.98e-04 | 1414.9ms/step | 45.2 samples/s
+Step 500/10001 | Loss: 5.0142 | LR: 4.98e-04 | 1412.5ms/step | 45.3 samples/s
+Validation at step 500: Loss = 5.0181, Perplexity = 151.12
+Step 510/10001 | Loss: 4.9932 | LR: 4.98e-04 | 27734.7ms/step | 2.3 samples/s
+Step 520/10001 | Loss: 4.9800 | LR: 4.98e-04 | 1419.3ms/step | 45.1 samples/s
+Step 530/10001 | Loss: 4.9511 | LR: 4.98e-04 | 1418.1ms/step | 45.1 samples/s
+Step 540/10001 | Loss: 4.9066 | LR: 4.98e-04 | 1420.1ms/step | 45.1 samples/s
+Step 550/10001 | Loss: 4.8840 | LR: 4.97e-04 | 1421.7ms/step | 45.0 samples/s
+Step 560/10001 | Loss: 4.8907 | LR: 4.97e-04 | 1422.3ms/step | 45.0 samples/s
+Step 570/10001 | Loss: 4.8460 | LR: 4.97e-04 | 1418.1ms/step | 45.1 samples/s
+Step 580/10001 | Loss: 4.8418 | LR: 4.97e-04 | 1419.0ms/step | 45.1 samples/s
+Step 590/10001 | Loss: 4.8046 | LR: 4.97e-04 | 1419.7ms/step | 45.1 samples/s
+Step 600/10001 | Loss: 4.7993 | LR: 4.97e-04 | 1418.3ms/step | 45.1 samples/s
+Step 610/10001 | Loss: 4.7780 | LR: 4.97e-04 | 1412.6ms/step | 45.3 samples/s
+Step 620/10001 | Loss: 4.7542 | LR: 4.97e-04 | 1414.1ms/step | 45.3 samples/s
+Step 630/10001 | Loss: 4.7317 | LR: 4.96e-04 | 1418.0ms/step | 45.1 samples/s
+Step 640/10001 | Loss: 4.7007 | LR: 4.96e-04 | 1414.2ms/step | 45.3 samples/s
+Step 650/10001 | Loss: 4.6737 | LR: 4.96e-04 | 1411.0ms/step | 45.4 samples/s
+Step 660/10001 | Loss: 4.6989 | LR: 4.96e-04 | 1412.9ms/step | 45.3 samples/s
+Step 670/10001 | Loss: 4.6503 | LR: 4.96e-04 | 1514.2ms/step | 42.3 samples/s
+Step 680/10001 | Loss: 4.6321 | LR: 4.96e-04 | 1415.9ms/step | 45.2 samples/s
+Step 690/10001 | Loss: 4.6195 | LR: 4.96e-04 | 1419.8ms/step | 45.1 samples/s
+Step 700/10001 | Loss: 4.6421 | LR: 4.95e-04 | 1418.5ms/step | 45.1 samples/s
+Step 710/10001 | Loss: 4.6036 | LR: 4.95e-04 | 1417.8ms/step | 45.1 samples/s
+Step 720/10001 | Loss: 4.6091 | LR: 4.95e-04 | 1420.2ms/step | 45.1 samples/s
+Step 730/10001 | Loss: 4.5887 | LR: 4.95e-04 | 1417.6ms/step | 45.1 samples/s
+Step 740/10001 | Loss: 4.5742 | LR: 4.95e-04 | 1418.1ms/step | 45.1 samples/s
+Step 750/10001 | Loss: 4.5564 | LR: 4.95e-04 | 1420.1ms/step | 45.1 samples/s
+Validation at step 750: Loss = 4.5407, Perplexity = 93.76
+Step 760/10001 | Loss: 4.5113 | LR: 4.95e-04 | 27714.1ms/step | 2.3 samples/s
+Step 770/10001 | Loss: 4.5527 | LR: 4.94e-04 | 1420.6ms/step | 45.1 samples/s
+Step 780/10001 | Loss: 4.4909 | LR: 4.94e-04 | 1421.1ms/step | 45.0 samples/s
+Step 790/10001 | Loss: 4.4981 | LR: 4.94e-04 | 1421.2ms/step | 45.0 samples/s
+Step 800/10001 | Loss: 4.5003 | LR: 4.94e-04 | 1423.2ms/step | 45.0 samples/s
+Step 810/10001 | Loss: 4.4721 | LR: 4.94e-04 | 1419.0ms/step | 45.1 samples/s
+Step 820/10001 | Loss: 4.4419 | LR: 4.94e-04 | 1413.0ms/step | 45.3 samples/s
+Step 830/10001 | Loss: 4.4586 | LR: 4.93e-04 | 1413.4ms/step | 45.3 samples/s
+Step 840/10001 | Loss: 4.4202 | LR: 4.93e-04 | 1414.3ms/step | 45.3 samples/s
+Step 850/10001 | Loss: 4.4399 | LR: 4.93e-04 | 1418.0ms/step | 45.1 samples/s
+Step 860/10001 | Loss: 4.4019 | LR: 4.93e-04 | 1416.9ms/step | 45.2 samples/s
+Step 870/10001 | Loss: 4.3893 | LR: 4.93e-04 | 1417.8ms/step | 45.1 samples/s
+Step 880/10001 | Loss: 4.3968 | LR: 4.92e-04 | 1413.3ms/step | 45.3 samples/s
+Step 890/10001 | Loss: 4.4044 | LR: 4.92e-04 | 1415.4ms/step | 45.2 samples/s
+Step 900/10001 | Loss: 4.3605 | LR: 4.92e-04 | 1417.1ms/step | 45.2 samples/s
+Step 910/10001 | Loss: 4.3812 | LR: 4.92e-04 | 1417.9ms/step | 45.1 samples/s
+Step 920/10001 | Loss: 4.3353 | LR: 4.92e-04 | 1420.0ms/step | 45.1 samples/s
+Step 930/10001 | Loss: 4.3438 | LR: 4.91e-04 | 1423.9ms/step | 44.9 samples/s
+Step 940/10001 | Loss: 4.3485 | LR: 4.91e-04 | 1425.8ms/step | 44.9 samples/s
+Step 950/10001 | Loss: 4.3144 | LR: 4.91e-04 | 1428.0ms/step | 44.8 samples/s
+Step 960/10001 | Loss: 4.2835 | LR: 4.91e-04 | 1421.9ms/step | 45.0 samples/s
+Step 970/10001 | Loss: 4.3012 | LR: 4.91e-04 | 1412.0ms/step | 45.3 samples/s
+Step 980/10001 | Loss: 4.2622 | LR: 4.90e-04 | 1412.1ms/step | 45.3 samples/s
+Step 990/10001 | Loss: 4.2926 | LR: 4.90e-04 | 1416.2ms/step | 45.2 samples/s
+Step 1000/10001 | Loss: 4.2968 | LR: 4.90e-04 | 1418.4ms/step | 45.1 samples/s
+Validation at step 1000: Loss = 4.2850, Perplexity = 72.60
+Step 1010/10001 | Loss: 4.2583 | LR: 4.90e-04 | 27705.7ms/step | 2.3 samples/s
+Step 1020/10001 | Loss: 4.2810 | LR: 4.89e-04 | 1420.0ms/step | 45.1 samples/s
+Step 1030/10001 | Loss: 4.2446 | LR: 4.89e-04 | 1420.5ms/step | 45.1 samples/s
+Step 1040/10001 | Loss: 4.2266 | LR: 4.89e-04 | 1419.5ms/step | 45.1 samples/s
+Step 1050/10001 | Loss: 4.2444 | LR: 4.89e-04 | 1419.9ms/step | 45.1 samples/s
+Step 1060/10001 | Loss: 4.2556 | LR: 4.88e-04 | 1419.1ms/step | 45.1 samples/s
+Step 1070/10001 | Loss: 4.2651 | LR: 4.88e-04 | 1418.7ms/step | 45.1 samples/s
+Step 1080/10001 | Loss: 4.2048 | LR: 4.88e-04 | 1417.2ms/step | 45.2 samples/s
+Step 1090/10001 | Loss: 4.2114 | LR: 4.88e-04 | 1412.9ms/step | 45.3 samples/s
+Step 1100/10001 | Loss: 4.2000 | LR: 4.88e-04 | 1416.1ms/step | 45.2 samples/s
+Step 1110/10001 | Loss: 4.2127 | LR: 4.87e-04 | 1418.4ms/step | 45.1 samples/s
+Step 1120/10001 | Loss: 4.1903 | LR: 4.87e-04 | 1423.5ms/step | 45.0 samples/s
+Step 1130/10001 | Loss: 4.2056 | LR: 4.87e-04 | 1420.4ms/step | 45.1 samples/s
+Step 1140/10001 | Loss: 4.2039 | LR: 4.87e-04 | 1418.0ms/step | 45.1 samples/s
+Step 1150/10001 | Loss: 4.1674 | LR: 4.86e-04 | 1417.4ms/step | 45.2 samples/s
+Step 1160/10001 | Loss: 4.1720 | LR: 4.86e-04 | 1418.1ms/step | 45.1 samples/s
+Step 1170/10001 | Loss: 4.1412 | LR: 4.86e-04 | 1420.3ms/step | 45.1 samples/s
+Step 1180/10001 | Loss: 4.1421 | LR: 4.85e-04 | 1422.0ms/step | 45.0 samples/s
+Step 1190/10001 | Loss: 4.1693 | LR: 4.85e-04 | 1421.7ms/step | 45.0 samples/s
+Step 1200/10001 | Loss: 4.1387 | LR: 4.85e-04 | 1419.7ms/step | 45.1 samples/s
+Step 1210/10001 | Loss: 4.1739 | LR: 4.85e-04 | 1413.6ms/step | 45.3 samples/s
+Step 1220/10001 | Loss: 4.1706 | LR: 4.84e-04 | 1409.0ms/step | 45.4 samples/s
+Step 1230/10001 | Loss: 4.1566 | LR: 4.84e-04 | 1411.5ms/step | 45.3 samples/s
+Step 1240/10001 | Loss: 4.1440 | LR: 4.84e-04 | 1417.8ms/step | 45.1 samples/s
+Step 1250/10001 | Loss: 4.1263 | LR: 4.84e-04 | 1418.8ms/step | 45.1 samples/s
+Validation at step 1250: Loss = 4.1136, Perplexity = 61.16
+Step 1260/10001 | Loss: 4.1520 | LR: 4.83e-04 | 27717.8ms/step | 2.3 samples/s
+Step 1270/10001 | Loss: 4.0992 | LR: 4.83e-04 | 1419.8ms/step | 45.1 samples/s
+Step 1280/10001 | Loss: 4.0874 | LR: 4.83e-04 | 1417.7ms/step | 45.1 samples/s
+Step 1290/10001 | Loss: 4.1115 | LR: 4.82e-04 | 1414.1ms/step | 45.3 samples/s
+Step 1300/10001 | Loss: 4.1040 | LR: 4.82e-04 | 1417.6ms/step | 45.1 samples/s
+Step 1310/10001 | Loss: 4.1012 | LR: 4.82e-04 | 1418.2ms/step | 45.1 samples/s
+Step 1320/10001 | Loss: 4.1052 | LR: 4.82e-04 | 1411.0ms/step | 45.4 samples/s
+Step 1330/10001 | Loss: 4.0979 | LR: 4.81e-04 | 1415.3ms/step | 45.2 samples/s
+Step 1340/10001 | Loss: 4.1017 | LR: 4.81e-04 | 1417.9ms/step | 45.1 samples/s
+Step 1350/10001 | Loss: 4.0697 | LR: 4.81e-04 | 1419.6ms/step | 45.1 samples/s
+Step 1360/10001 | Loss: 4.0711 | LR: 4.80e-04 | 1417.9ms/step | 45.1 samples/s
+Step 1370/10001 | Loss: 4.0501 | LR: 4.80e-04 | 1418.7ms/step | 45.1 samples/s
+Step 1380/10001 | Loss: 4.0408 | LR: 4.80e-04 | 1420.9ms/step | 45.0 samples/s
+Step 1390/10001 | Loss: 4.0390 | LR: 4.79e-04 | 1423.4ms/step | 45.0 samples/s
+Step 1400/10001 | Loss: 3.9923 | LR: 4.79e-04 | 1422.5ms/step | 45.0 samples/s
+Step 1410/10001 | Loss: 4.0468 | LR: 4.79e-04 | 1419.8ms/step | 45.1 samples/s
+Step 1420/10001 | Loss: 4.0262 | LR: 4.78e-04 | 1504.2ms/step | 42.5 samples/s
+Step 1430/10001 | Loss: 4.0427 | LR: 4.78e-04 | 1417.7ms/step | 45.1 samples/s
+Step 1440/10001 | Loss: 4.0272 | LR: 4.78e-04 | 1415.0ms/step | 45.2 samples/s
+Step 1450/10001 | Loss: 4.0284 | LR: 4.77e-04 | 1418.6ms/step | 45.1 samples/s
+Step 1460/10001 | Loss: 4.0452 | LR: 4.77e-04 | 1420.5ms/step | 45.1 samples/s
+Step 1470/10001 | Loss: 4.0209 | LR: 4.77e-04 | 1421.6ms/step | 45.0 samples/s
+Step 1480/10001 | Loss: 3.9727 | LR: 4.76e-04 | 1420.2ms/step | 45.1 samples/s
+Step 1490/10001 | Loss: 4.0104 | LR: 4.76e-04 | 1417.7ms/step | 45.1 samples/s
+Step 1500/10001 | Loss: 3.9912 | LR: 4.76e-04 | 1413.7ms/step | 45.3 samples/s
+Validation at step 1500: Loss = 3.9991, Perplexity = 54.55
+Step 1510/10001 | Loss: 4.0049 | LR: 4.75e-04 | 27682.1ms/step | 2.3 samples/s
+Step 1520/10001 | Loss: 3.9804 | LR: 4.75e-04 | 1419.5ms/step | 45.1 samples/s
+Step 1530/10001 | Loss: 3.9945 | LR: 4.75e-04 | 1417.1ms/step | 45.2 samples/s
+Step 1540/10001 | Loss: 3.9767 | LR: 4.74e-04 | 1417.8ms/step | 45.1 samples/s
+Step 1550/10001 | Loss: 3.9590 | LR: 4.74e-04 | 1418.5ms/step | 45.1 samples/s
+Step 1560/10001 | Loss: 3.9838 | LR: 4.74e-04 | 1416.0ms/step | 45.2 samples/s
+Step 1570/10001 | Loss: 3.9701 | LR: 4.73e-04 | 1419.3ms/step | 45.1 samples/s
+Step 1580/10001 | Loss: 3.9553 | LR: 4.73e-04 | 1418.1ms/step | 45.1 samples/s
+Step 1590/10001 | Loss: 3.9755 | LR: 4.73e-04 | 1413.5ms/step | 45.3 samples/s
+Step 1600/10001 | Loss: 3.9495 | LR: 4.72e-04 | 1412.3ms/step | 45.3 samples/s
+Step 1610/10001 | Loss: 3.9375 | LR: 4.72e-04 | 1416.0ms/step | 45.2 samples/s
+Step 1620/10001 | Loss: 3.9546 | LR: 4.71e-04 | 1417.9ms/step | 45.1 samples/s
+Step 1630/10001 | Loss: 3.9382 | LR: 4.71e-04 | 1420.8ms/step | 45.0 samples/s
+Step 1640/10001 | Loss: 3.9262 | LR: 4.71e-04 | 1419.3ms/step | 45.1 samples/s
+Step 1650/10001 | Loss: 3.9288 | LR: 4.70e-04 | 1418.4ms/step | 45.1 samples/s
+Step 1660/10001 | Loss: 3.9509 | LR: 4.70e-04 | 1418.6ms/step | 45.1 samples/s
+Step 1670/10001 | Loss: 3.9104 | LR: 4.70e-04 | 1414.4ms/step | 45.2 samples/s
+Step 1680/10001 | Loss: 3.9385 | LR: 4.69e-04 | 1412.6ms/step | 45.3 samples/s
+Step 1690/10001 | Loss: 3.9186 | LR: 4.69e-04 | 1415.4ms/step | 45.2 samples/s
+Step 1700/10001 | Loss: 3.9234 | LR: 4.68e-04 | 1417.0ms/step | 45.2 samples/s
+Step 1710/10001 | Loss: 3.9089 | LR: 4.68e-04 | 1420.1ms/step | 45.1 samples/s
+Step 1720/10001 | Loss: 3.9051 | LR: 4.68e-04 | 1419.8ms/step | 45.1 samples/s
+Step 1730/10001 | Loss: 3.9341 | LR: 4.67e-04 | 1418.2ms/step | 45.1 samples/s
+Step 1740/10001 | Loss: 3.9070 | LR: 4.67e-04 | 1419.7ms/step | 45.1 samples/s
+Step 1750/10001 | Loss: 3.9380 | LR: 4.67e-04 | 1420.5ms/step | 45.1 samples/s
+Validation at step 1750: Loss = 3.9121, Perplexity = 50.00
+Step 1760/10001 | Loss: 3.9248 | LR: 4.66e-04 | 27719.2ms/step | 2.3 samples/s
+Step 1770/10001 | Loss: 3.8785 | LR: 4.66e-04 | 1412.3ms/step | 45.3 samples/s
+Step 1780/10001 | Loss: 3.8947 | LR: 4.65e-04 | 1412.5ms/step | 45.3 samples/s
+Step 1790/10001 | Loss: 3.9096 | LR: 4.65e-04 | 1419.8ms/step | 45.1 samples/s
+Step 1800/10001 | Loss: 3.8994 | LR: 4.65e-04 | 1417.6ms/step | 45.1 samples/s
+Step 1810/10001 | Loss: 3.8840 | LR: 4.64e-04 | 1418.3ms/step | 45.1 samples/s
+Step 1820/10001 | Loss: 3.8719 | LR: 4.64e-04 | 1420.8ms/step | 45.0 samples/s
+Step 1830/10001 | Loss: 3.8987 | LR: 4.63e-04 | 1417.1ms/step | 45.2 samples/s
+Step 1840/10001 | Loss: 3.8635 | LR: 4.63e-04 | 1412.6ms/step | 45.3 samples/s
+Step 1850/10001 | Loss: 3.8736 | LR: 4.62e-04 | 1410.6ms/step | 45.4 samples/s
+Step 1860/10001 | Loss: 3.8749 | LR: 4.62e-04 | 1410.7ms/step | 45.4 samples/s
+Step 1870/10001 | Loss: 3.8714 | LR: 4.62e-04 | 1416.4ms/step | 45.2 samples/s
+Step 1880/10001 | Loss: 3.8721 | LR: 4.61e-04 | 1416.4ms/step | 45.2 samples/s
+Step 1890/10001 | Loss: 3.8607 | LR: 4.61e-04 | 1419.3ms/step | 45.1 samples/s
+Step 1900/10001 | Loss: 3.8791 | LR: 4.60e-04 | 1419.8ms/step | 45.1 samples/s
+Step 1910/10001 | Loss: 3.8709 | LR: 4.60e-04 | 1414.1ms/step | 45.3 samples/s
+Step 1920/10001 | Loss: 3.8791 | LR: 4.59e-04 | 1411.2ms/step | 45.4 samples/s
+Step 1930/10001 | Loss: 3.8673 | LR: 4.59e-04 | 1413.7ms/step | 45.3 samples/s
+Step 1940/10001 | Loss: 3.8463 | LR: 4.59e-04 | 1415.7ms/step | 45.2 samples/s
+Step 1950/10001 | Loss: 3.8558 | LR: 4.58e-04 | 1417.0ms/step | 45.2 samples/s
+Step 1960/10001 | Loss: 3.8597 | LR: 4.58e-04 | 1418.5ms/step | 45.1 samples/s
+Step 1970/10001 | Loss: 3.8368 | LR: 4.57e-04 | 1419.9ms/step | 45.1 samples/s
+Step 1980/10001 | Loss: 3.8547 | LR: 4.57e-04 | 1417.2ms/step | 45.2 samples/s
+Step 1990/10001 | Loss: 3.8505 | LR: 4.56e-04 | 1416.7ms/step | 45.2 samples/s
+Step 2000/10001 | Loss: 3.8289 | LR: 4.56e-04 | 1417.8ms/step | 45.1 samples/s
+Validation at step 2000: Loss = 3.8415, Perplexity = 46.60
+Step 2010/10001 | Loss: 3.8347 | LR: 4.55e-04 | 27664.5ms/step | 2.3 samples/s
+Step 2020/10001 | Loss: 3.8602 | LR: 4.55e-04 | 1417.3ms/step | 45.2 samples/s
+Step 2030/10001 | Loss: 3.8255 | LR: 4.55e-04 | 1418.3ms/step | 45.1 samples/s
+Step 2040/10001 | Loss: 3.8107 | LR: 4.54e-04 | 1420.1ms/step | 45.1 samples/s
+Step 2050/10001 | Loss: 3.8340 | LR: 4.54e-04 | 1416.4ms/step | 45.2 samples/s
+Step 2060/10001 | Loss: 3.8298 | LR: 4.53e-04 | 1412.6ms/step | 45.3 samples/s
+Step 2070/10001 | Loss: 3.8115 | LR: 4.53e-04 | 1416.2ms/step | 45.2 samples/s
+Step 2080/10001 | Loss: 3.8579 | LR: 4.52e-04 | 1414.8ms/step | 45.2 samples/s
+Step 2090/10001 | Loss: 3.8096 | LR: 4.52e-04 | 1415.5ms/step | 45.2 samples/s
+Step 2100/10001 | Loss: 3.8110 | LR: 4.51e-04 | 1419.0ms/step | 45.1 samples/s
+Step 2110/10001 | Loss: 3.7937 | LR: 4.51e-04 | 1420.0ms/step | 45.1 samples/s
+Step 2120/10001 | Loss: 3.8188 | LR: 4.50e-04 | 1420.2ms/step | 45.1 samples/s
+Step 2130/10001 | Loss: 3.7939 | LR: 4.50e-04 | 1419.8ms/step | 45.1 samples/s
+Step 2140/10001 | Loss: 3.8144 | LR: 4.49e-04 | 1418.9ms/step | 45.1 samples/s
+Step 2150/10001 | Loss: 3.7971 | LR: 4.49e-04 | 1413.4ms/step | 45.3 samples/s
+Step 2160/10001 | Loss: 3.7906 | LR: 4.48e-04 | 1409.7ms/step | 45.4 samples/s
+Step 2170/10001 | Loss: 3.8110 | LR: 4.48e-04 | 1500.4ms/step | 42.7 samples/s
+Step 2180/10001 | Loss: 3.8090 | LR: 4.48e-04 | 1419.6ms/step | 45.1 samples/s
+Step 2190/10001 | Loss: 3.8026 | LR: 4.47e-04 | 1416.1ms/step | 45.2 samples/s
+Step 2200/10001 | Loss: 3.8092 | LR: 4.47e-04 | 1413.6ms/step | 45.3 samples/s
+Step 2210/10001 | Loss: 3.7613 | LR: 4.46e-04 | 1416.3ms/step | 45.2 samples/s
+Step 2220/10001 | Loss: 3.8035 | LR: 4.46e-04 | 1413.8ms/step | 45.3 samples/s
+Step 2230/10001 | Loss: 3.8072 | LR: 4.45e-04 | 1417.0ms/step | 45.2 samples/s
+Step 2240/10001 | Loss: 3.7765 | LR: 4.45e-04 | 1416.9ms/step | 45.2 samples/s
+Step 2250/10001 | Loss: 3.7421 | LR: 4.44e-04 | 1416.3ms/step | 45.2 samples/s
+Validation at step 2250: Loss = 3.7824, Perplexity = 43.92
+Step 2260/10001 | Loss: 3.7841 | LR: 4.44e-04 | 27731.0ms/step | 2.3 samples/s
+Step 2270/10001 | Loss: 3.8006 | LR: 4.43e-04 | 1418.9ms/step | 45.1 samples/s
+Step 2280/10001 | Loss: 3.7716 | LR: 4.43e-04 | 1421.0ms/step | 45.0 samples/s
+Step 2290/10001 | Loss: 3.8004 | LR: 4.42e-04 | 1417.9ms/step | 45.1 samples/s
+Step 2300/10001 | Loss: 3.8101 | LR: 4.42e-04 | 1416.1ms/step | 45.2 samples/s
+Step 2310/10001 | Loss: 3.7931 | LR: 4.41e-04 | 1419.2ms/step | 45.1 samples/s
+Step 2320/10001 | Loss: 3.7598 | LR: 4.40e-04 | 1419.4ms/step | 45.1 samples/s
+Step 2330/10001 | Loss: 3.7563 | LR: 4.40e-04 | 1418.2ms/step | 45.1 samples/s
+Step 2340/10001 | Loss: 3.7694 | LR: 4.39e-04 | 1415.3ms/step | 45.2 samples/s
+Step 2350/10001 | Loss: 3.7911 | LR: 4.39e-04 | 1416.5ms/step | 45.2 samples/s
+Step 2360/10001 | Loss: 3.7689 | LR: 4.38e-04 | 1416.0ms/step | 45.2 samples/s
+Step 2370/10001 | Loss: 3.7768 | LR: 4.38e-04 | 1414.0ms/step | 45.3 samples/s
+Step 2380/10001 | Loss: 3.7598 | LR: 4.37e-04 | 1416.4ms/step | 45.2 samples/s
+Step 2390/10001 | Loss: 3.7359 | LR: 4.37e-04 | 1415.0ms/step | 45.2 samples/s
+Step 2400/10001 | Loss: 3.7514 | LR: 4.36e-04 | 1409.5ms/step | 45.4 samples/s
+Step 2410/10001 | Loss: 3.7736 | LR: 4.36e-04 | 1411.1ms/step | 45.4 samples/s
+Step 2420/10001 | Loss: 3.7489 | LR: 4.35e-04 | 1416.1ms/step | 45.2 samples/s
+Step 2430/10001 | Loss: 3.7408 | LR: 4.35e-04 | 1416.2ms/step | 45.2 samples/s
+Step 2440/10001 | Loss: 3.7366 | LR: 4.34e-04 | 1416.0ms/step | 45.2 samples/s
+Step 2450/10001 | Loss: 3.7464 | LR: 4.34e-04 | 1417.2ms/step | 45.2 samples/s
+Step 2460/10001 | Loss: 3.7315 | LR: 4.33e-04 | 1419.0ms/step | 45.1 samples/s
+Step 2470/10001 | Loss: 3.7599 | LR: 4.33e-04 | 1413.7ms/step | 45.3 samples/s
+Step 2480/10001 | Loss: 3.7553 | LR: 4.32e-04 | 1409.6ms/step | 45.4 samples/s
+Step 2490/10001 | Loss: 3.7381 | LR: 4.31e-04 | 1410.4ms/step | 45.4 samples/s
+Step 2500/10001 | Loss: 3.7415 | LR: 4.31e-04 | 1415.9ms/step | 45.2 samples/s
+Validation at step 2500: Loss = 3.7354, Perplexity = 41.91
+Saved checkpoint to checkpoints/run_20260430_021105/checkpoint-2500
+Step 2510/10001 | Loss: 3.7253 | LR: 4.30e-04 | 28180.4ms/step | 2.3 samples/s
+Step 2520/10001 | Loss: 3.7040 | LR: 4.30e-04 | 1414.9ms/step | 45.2 samples/s
+Step 2530/10001 | Loss: 3.7281 | LR: 4.29e-04 | 1414.7ms/step | 45.2 samples/s
+Step 2540/10001 | Loss: 3.7288 | LR: 4.29e-04 | 1418.3ms/step | 45.1 samples/s
+Step 2550/10001 | Loss: 3.7454 | LR: 4.28e-04 | 1417.7ms/step | 45.1 samples/s
+Step 2560/10001 | Loss: 3.6841 | LR: 4.28e-04 | 1417.0ms/step | 45.2 samples/s
+Step 2570/10001 | Loss: 3.7227 | LR: 4.27e-04 | 1416.5ms/step | 45.2 samples/s
+Step 2580/10001 | Loss: 3.7134 | LR: 4.27e-04 | 1415.7ms/step | 45.2 samples/s
+Step 2590/10001 | Loss: 3.7240 | LR: 4.26e-04 | 1417.4ms/step | 45.2 samples/s
+Step 2600/10001 | Loss: 3.7045 | LR: 4.25e-04 | 1422.5ms/step | 45.0 samples/s
+Step 2610/10001 | Loss: 3.7506 | LR: 4.25e-04 | 1423.6ms/step | 45.0 samples/s
+Step 2620/10001 | Loss: 3.7044 | LR: 4.24e-04 | 1419.4ms/step | 45.1 samples/s
+Step 2630/10001 | Loss: 3.6967 | LR: 4.24e-04 | 1416.7ms/step | 45.2 samples/s
+Step 2640/10001 | Loss: 3.7017 | LR: 4.23e-04 | 1416.3ms/step | 45.2 samples/s
+Step 2650/10001 | Loss: 3.6794 | LR: 4.23e-04 | 1418.5ms/step | 45.1 samples/s
+Step 2660/10001 | Loss: 3.7126 | LR: 4.22e-04 | 1419.5ms/step | 45.1 samples/s
+Step 2670/10001 | Loss: 3.7378 | LR: 4.21e-04 | 1419.6ms/step | 45.1 samples/s
+Step 2680/10001 | Loss: 3.6998 | LR: 4.21e-04 | 1420.9ms/step | 45.0 samples/s
+Step 2690/10001 | Loss: 3.6777 | LR: 4.20e-04 | 1418.1ms/step | 45.1 samples/s
+Step 2700/10001 | Loss: 3.6981 | LR: 4.20e-04 | 1410.2ms/step | 45.4 samples/s
+Step 2710/10001 | Loss: 3.6915 | LR: 4.19e-04 | 1410.8ms/step | 45.4 samples/s
+Step 2720/10001 | Loss: 3.6714 | LR: 4.18e-04 | 1413.7ms/step | 45.3 samples/s
+Step 2730/10001 | Loss: 3.6860 | LR: 4.18e-04 | 1419.8ms/step | 45.1 samples/s
+Step 2740/10001 | Loss: 3.6996 | LR: 4.17e-04 | 1420.0ms/step | 45.1 samples/s
+Step 2750/10001 | Loss: 3.7102 | LR: 4.17e-04 | 1420.2ms/step | 45.1 samples/s
+Validation at step 2750: Loss = 3.6935, Perplexity = 40.19
+Step 2760/10001 | Loss: 3.7028 | LR: 4.16e-04 | 27707.4ms/step | 2.3 samples/s
+Step 2770/10001 | Loss: 3.7125 | LR: 4.16e-04 | 1419.8ms/step | 45.1 samples/s
+Step 2780/10001 | Loss: 3.6811 | LR: 4.15e-04 | 1420.2ms/step | 45.1 samples/s
+Step 2790/10001 | Loss: 3.7134 | LR: 4.14e-04 | 1420.4ms/step | 45.1 samples/s
+Step 2800/10001 | Loss: 3.6869 | LR: 4.14e-04 | 1417.4ms/step | 45.2 samples/s
+Step 2810/10001 | Loss: 3.6634 | LR: 4.13e-04 | 1421.2ms/step | 45.0 samples/s
+Step 2820/10001 | Loss: 3.6885 | LR: 4.13e-04 | 1418.9ms/step | 45.1 samples/s
+Step 2830/10001 | Loss: 3.6949 | LR: 4.12e-04 | 1415.3ms/step | 45.2 samples/s
+Step 2840/10001 | Loss: 3.6870 | LR: 4.11e-04 | 1414.2ms/step | 45.3 samples/s
+Step 2850/10001 | Loss: 3.6776 | LR: 4.11e-04 | 1415.5ms/step | 45.2 samples/s
+Step 2860/10001 | Loss: 3.6810 | LR: 4.10e-04 | 1417.1ms/step | 45.2 samples/s
+Step 2870/10001 | Loss: 3.7004 | LR: 4.09e-04 | 1418.2ms/step | 45.1 samples/s
+Step 2880/10001 | Loss: 3.7016 | LR: 4.09e-04 | 1420.8ms/step | 45.0 samples/s
+Step 2890/10001 | Loss: 3.7040 | LR: 4.08e-04 | 1420.5ms/step | 45.1 samples/s
+Step 2900/10001 | Loss: 3.6583 | LR: 4.08e-04 | 1412.8ms/step | 45.3 samples/s
+Step 2910/10001 | Loss: 3.6607 | LR: 4.07e-04 | 1499.6ms/step | 42.7 samples/s
+Step 2920/10001 | Loss: 3.6733 | LR: 4.06e-04 | 1417.1ms/step | 45.2 samples/s
+Step 2930/10001 | Loss: 3.6819 | LR: 4.06e-04 | 1417.4ms/step | 45.2 samples/s
+Step 2940/10001 | Loss: 3.6392 | LR: 4.05e-04 | 1419.2ms/step | 45.1 samples/s
+Step 2950/10001 | Loss: 3.6443 | LR: 4.05e-04 | 1418.1ms/step | 45.1 samples/s
+Step 2960/10001 | Loss: 3.6571 | LR: 4.04e-04 | 1413.2ms/step | 45.3 samples/s
+Step 2970/10001 | Loss: 3.6617 | LR: 4.03e-04 | 1410.2ms/step | 45.4 samples/s
+Step 2980/10001 | Loss: 3.6638 | LR: 4.03e-04 | 1412.5ms/step | 45.3 samples/s
+Step 2990/10001 | Loss: 3.6721 | LR: 4.02e-04 | 1414.2ms/step | 45.3 samples/s
+Step 3000/10001 | Loss: 3.6472 | LR: 4.01e-04 | 1416.2ms/step | 45.2 samples/s
+Validation at step 3000: Loss = 3.6570, Perplexity = 38.74
+Step 3010/10001 | Loss: 3.6919 | LR: 4.01e-04 | 27684.8ms/step | 2.3 samples/s
+Step 3020/10001 | Loss: 3.6677 | LR: 4.00e-04 | 1408.5ms/step | 45.4 samples/s
+Step 3030/10001 | Loss: 3.6514 | LR: 4.00e-04 | 1410.1ms/step | 45.4 samples/s
+Step 3040/10001 | Loss: 3.6336 | LR: 3.99e-04 | 1414.6ms/step | 45.2 samples/s
+Step 3050/10001 | Loss: 3.6483 | LR: 3.98e-04 | 1418.0ms/step | 45.1 samples/s
+Step 3060/10001 | Loss: 3.6573 | LR: 3.98e-04 | 1420.4ms/step | 45.1 samples/s
+Step 3070/10001 | Loss: 3.6161 | LR: 3.97e-04 | 1421.6ms/step | 45.0 samples/s
+Step 3080/10001 | Loss: 3.6270 | LR: 3.96e-04 | 1421.6ms/step | 45.0 samples/s
+Step 3090/10001 | Loss: 3.6524 | LR: 3.96e-04 | 1421.3ms/step | 45.0 samples/s
+Step 3100/10001 | Loss: 3.6244 | LR: 3.95e-04 | 1412.5ms/step | 45.3 samples/s
+Step 3110/10001 | Loss: 3.6729 | LR: 3.94e-04 | 1409.9ms/step | 45.4 samples/s
+Step 3120/10001 | Loss: 3.6569 | LR: 3.94e-04 | 1410.6ms/step | 45.4 samples/s
+Step 3130/10001 | Loss: 3.6319 | LR: 3.93e-04 | 1412.8ms/step | 45.3 samples/s
+Step 3140/10001 | Loss: 3.6442 | LR: 3.92e-04 | 1416.2ms/step | 45.2 samples/s
+Step 3150/10001 | Loss: 3.6395 | LR: 3.92e-04 | 1416.7ms/step | 45.2 samples/s
+Step 3160/10001 | Loss: 3.6084 | LR: 3.91e-04 | 1417.8ms/step | 45.1 samples/s
+Step 3170/10001 | Loss: 3.6116 | LR: 3.90e-04 | 1417.5ms/step | 45.2 samples/s
+Step 3180/10001 | Loss: 3.6339 | LR: 3.90e-04 | 1411.8ms/step | 45.3 samples/s
+Step 3190/10001 | Loss: 3.6512 | LR: 3.89e-04 | 1410.9ms/step | 45.4 samples/s
+Step 3200/10001 | Loss: 3.6197 | LR: 3.89e-04 | 1415.2ms/step | 45.2 samples/s
+Step 3210/10001 | Loss: 3.6143 | LR: 3.88e-04 | 1414.8ms/step | 45.2 samples/s
+Step 3220/10001 | Loss: 3.6362 | LR: 3.87e-04 | 1417.6ms/step | 45.1 samples/s
+Step 3230/10001 | Loss: 3.6188 | LR: 3.87e-04 | 1417.5ms/step | 45.2 samples/s
+Step 3240/10001 | Loss: 3.6089 | LR: 3.86e-04 | 1416.8ms/step | 45.2 samples/s
+Step 3250/10001 | Loss: 3.6139 | LR: 3.85e-04 | 1418.5ms/step | 45.1 samples/s
+Validation at step 3250: Loss = 3.6240, Perplexity = 37.49
+Step 3260/10001 | Loss: 3.6326 | LR: 3.85e-04 | 27693.9ms/step | 2.3 samples/s
+Step 3270/10001 | Loss: 3.6180 | LR: 3.84e-04 | 1418.1ms/step | 45.1 samples/s
+Step 3280/10001 | Loss: 3.6243 | LR: 3.83e-04 | 1419.3ms/step | 45.1 samples/s
+Step 3290/10001 | Loss: 3.6063 | LR: 3.83e-04 | 1418.1ms/step | 45.1 samples/s
+Step 3300/10001 | Loss: 3.6312 | LR: 3.82e-04 | 1415.2ms/step | 45.2 samples/s
+Step 3310/10001 | Loss: 3.6165 | LR: 3.81e-04 | 1417.2ms/step | 45.2 samples/s
+Step 3320/10001 | Loss: 3.6066 | LR: 3.80e-04 | 1418.0ms/step | 45.1 samples/s
+Step 3330/10001 | Loss: 3.5785 | LR: 3.80e-04 | 1416.5ms/step | 45.2 samples/s
+Step 3340/10001 | Loss: 3.5920 | LR: 3.79e-04 | 1412.2ms/step | 45.3 samples/s
+Step 3350/10001 | Loss: 3.5889 | LR: 3.78e-04 | 1410.8ms/step | 45.4 samples/s
+Step 3360/10001 | Loss: 3.5970 | LR: 3.78e-04 | 1414.9ms/step | 45.2 samples/s
+Step 3370/10001 | Loss: 3.6109 | LR: 3.77e-04 | 1417.2ms/step | 45.2 samples/s
+Step 3380/10001 | Loss: 3.5885 | LR: 3.76e-04 | 1417.1ms/step | 45.2 samples/s
+Step 3390/10001 | Loss: 3.5969 | LR: 3.76e-04 | 1419.4ms/step | 45.1 samples/s
+Step 3400/10001 | Loss: 3.5767 | LR: 3.75e-04 | 1416.3ms/step | 45.2 samples/s
+Step 3410/10001 | Loss: 3.6015 | LR: 3.74e-04 | 1413.3ms/step | 45.3 samples/s
+Step 3420/10001 | Loss: 3.5913 | LR: 3.74e-04 | 1414.1ms/step | 45.3 samples/s
+Step 3430/10001 | Loss: 3.5771 | LR: 3.73e-04 | 1412.1ms/step | 45.3 samples/s
+Step 3440/10001 | Loss: 3.5826 | LR: 3.72e-04 | 1414.2ms/step | 45.3 samples/s
+Step 3450/10001 | Loss: 3.6032 | LR: 3.72e-04 | 1416.2ms/step | 45.2 samples/s
+Step 3460/10001 | Loss: 3.6048 | LR: 3.71e-04 | 1420.6ms/step | 45.1 samples/s
+Step 3470/10001 | Loss: 3.5787 | LR: 3.70e-04 | 1420.8ms/step | 45.0 samples/s
+Step 3480/10001 | Loss: 3.5868 | LR: 3.69e-04 | 1420.6ms/step | 45.1 samples/s
+Step 3490/10001 | Loss: 3.5830 | LR: 3.69e-04 | 1418.0ms/step | 45.1 samples/s
+Step 3500/10001 | Loss: 3.5627 | LR: 3.68e-04 | 1419.8ms/step | 45.1 samples/s
+Validation at step 3500: Loss = 3.5938, Perplexity = 36.37
+Step 3510/10001 | Loss: 3.5618 | LR: 3.67e-04 | 27731.7ms/step | 2.3 samples/s
+Step 3520/10001 | Loss: 3.5570 | LR: 3.67e-04 | 1418.2ms/step | 45.1 samples/s
+Step 3530/10001 | Loss: 3.6158 | LR: 3.66e-04 | 1420.8ms/step | 45.0 samples/s
+Step 3540/10001 | Loss: 3.5909 | LR: 3.65e-04 | 1423.2ms/step | 45.0 samples/s
+Step 3550/10001 | Loss: 3.6012 | LR: 3.65e-04 | 1423.2ms/step | 45.0 samples/s
+Step 3560/10001 | Loss: 3.5888 | LR: 3.64e-04 | 1423.2ms/step | 45.0 samples/s
+Step 3570/10001 | Loss: 3.5777 | LR: 3.63e-04 | 1421.0ms/step | 45.0 samples/s
+Step 3580/10001 | Loss: 3.5927 | LR: 3.62e-04 | 1419.0ms/step | 45.1 samples/s
+Step 3590/10001 | Loss: 3.5678 | LR: 3.62e-04 | 1417.0ms/step | 45.2 samples/s
+Step 3600/10001 | Loss: 3.5978 | LR: 3.61e-04 | 1418.8ms/step | 45.1 samples/s
+Step 3610/10001 | Loss: 3.5677 | LR: 3.60e-04 | 1417.9ms/step | 45.1 samples/s
+Step 3620/10001 | Loss: 3.5881 | LR: 3.60e-04 | 1418.3ms/step | 45.1 samples/s
+Step 3630/10001 | Loss: 3.5609 | LR: 3.59e-04 | 1416.2ms/step | 45.2 samples/s
+Step 3640/10001 | Loss: 3.5917 | LR: 3.58e-04 | 1416.7ms/step | 45.2 samples/s
+Step 3650/10001 | Loss: 3.5945 | LR: 3.57e-04 | 1501.0ms/step | 42.6 samples/s
+Step 3660/10001 | Loss: 3.5703 | LR: 3.57e-04 | 1418.5ms/step | 45.1 samples/s
+Step 3670/10001 | Loss: 3.5621 | LR: 3.56e-04 | 1415.6ms/step | 45.2 samples/s
+Step 3680/10001 | Loss: 3.5508 | LR: 3.55e-04 | 1416.6ms/step | 45.2 samples/s
+Step 3690/10001 | Loss: 3.5951 | LR: 3.55e-04 | 1417.2ms/step | 45.2 samples/s
+Step 3700/10001 | Loss: 3.5743 | LR: 3.54e-04 | 1419.9ms/step | 45.1 samples/s
+Step 3710/10001 | Loss: 3.5796 | LR: 3.53e-04 | 1420.5ms/step | 45.1 samples/s
+Step 3720/10001 | Loss: 3.5432 | LR: 3.52e-04 | 1421.2ms/step | 45.0 samples/s
+Step 3730/10001 | Loss: 3.5680 | LR: 3.52e-04 | 1418.3ms/step | 45.1 samples/s
+Step 3740/10001 | Loss: 3.5424 | LR: 3.51e-04 | 1418.9ms/step | 45.1 samples/s
+Step 3750/10001 | Loss: 3.5672 | LR: 3.50e-04 | 1421.8ms/step | 45.0 samples/s
+Validation at step 3750: Loss = 3.5663, Perplexity = 35.39
+Step 3760/10001 | Loss: 3.6109 | LR: 3.50e-04 | 27687.5ms/step | 2.3 samples/s
+Step 3770/10001 | Loss: 3.5806 | LR: 3.49e-04 | 1417.8ms/step | 45.1 samples/s
+Step 3780/10001 | Loss: 3.5982 | LR: 3.48e-04 | 1414.4ms/step | 45.3 samples/s
+Step 3790/10001 | Loss: 3.5395 | LR: 3.47e-04 | 1418.3ms/step | 45.1 samples/s
+Step 3800/10001 | Loss: 3.5539 | LR: 3.47e-04 | 1420.4ms/step | 45.1 samples/s
+Step 3810/10001 | Loss: 3.5518 | LR: 3.46e-04 | 1419.2ms/step | 45.1 samples/s
+Step 3820/10001 | Loss: 3.5244 | LR: 3.45e-04 | 1417.7ms/step | 45.1 samples/s
+Step 3830/10001 | Loss: 3.5633 | LR: 3.44e-04 | 1421.1ms/step | 45.0 samples/s
+Step 3840/10001 | Loss: 3.5786 | LR: 3.44e-04 | 1419.4ms/step | 45.1 samples/s
+Step 3850/10001 | Loss: 3.5503 | LR: 3.43e-04 | 1423.7ms/step | 45.0 samples/s
+Step 3860/10001 | Loss: 3.5553 | LR: 3.42e-04 | 1424.3ms/step | 44.9 samples/s
+Step 3870/10001 | Loss: 3.5425 | LR: 3.41e-04 | 1421.9ms/step | 45.0 samples/s
+Step 3880/10001 | Loss: 3.5836 | LR: 3.41e-04 | 1419.8ms/step | 45.1 samples/s
+Step 3890/10001 | Loss: 3.5571 | LR: 3.40e-04 | 1415.6ms/step | 45.2 samples/s
+Step 3900/10001 | Loss: 3.5454 | LR: 3.39e-04 | 1416.3ms/step | 45.2 samples/s
+Step 3910/10001 | Loss: 3.5698 | LR: 3.39e-04 | 1417.7ms/step | 45.1 samples/s
+Step 3920/10001 | Loss: 3.5412 | LR: 3.38e-04 | 1416.4ms/step | 45.2 samples/s
+Step 3930/10001 | Loss: 3.5134 | LR: 3.37e-04 | 1411.6ms/step | 45.3 samples/s
+Step 3940/10001 | Loss: 3.5424 | LR: 3.36e-04 | 1409.6ms/step | 45.4 samples/s
+Step 3950/10001 | Loss: 3.5444 | LR: 3.36e-04 | 1409.4ms/step | 45.4 samples/s
+Step 3960/10001 | Loss: 3.5554 | LR: 3.35e-04 | 1412.7ms/step | 45.3 samples/s
+Step 3970/10001 | Loss: 3.5437 | LR: 3.34e-04 | 1416.8ms/step | 45.2 samples/s
+Step 3980/10001 | Loss: 3.5707 | LR: 3.33e-04 | 1415.9ms/step | 45.2 samples/s
+Step 3990/10001 | Loss: 3.5338 | LR: 3.33e-04 | 1418.6ms/step | 45.1 samples/s
+Step 4000/10001 | Loss: 3.5162 | LR: 3.32e-04 | 1419.4ms/step | 45.1 samples/s
+Validation at step 4000: Loss = 3.5415, Perplexity = 34.52
+Step 4010/10001 | Loss: 3.5311 | LR: 3.31e-04 | 27694.5ms/step | 2.3 samples/s
+Step 4020/10001 | Loss: 3.5357 | LR: 3.30e-04 | 1417.7ms/step | 45.1 samples/s
+Step 4030/10001 | Loss: 3.5068 | LR: 3.30e-04 | 1413.8ms/step | 45.3 samples/s
+Step 4040/10001 | Loss: 3.5525 | LR: 3.29e-04 | 1410.1ms/step | 45.4 samples/s
+Step 4050/10001 | Loss: 3.5382 | LR: 3.28e-04 | 1416.4ms/step | 45.2 samples/s
+Step 4060/10001 | Loss: 3.5241 | LR: 3.27e-04 | 1417.5ms/step | 45.2 samples/s
+Step 4070/10001 | Loss: 3.5263 | LR: 3.27e-04 | 1414.1ms/step | 45.3 samples/s
+Step 4080/10001 | Loss: 3.5450 | LR: 3.26e-04 | 1409.6ms/step | 45.4 samples/s
+Step 4090/10001 | Loss: 3.5189 | LR: 3.25e-04 | 1412.5ms/step | 45.3 samples/s
+Step 4100/10001 | Loss: 3.5176 | LR: 3.24e-04 | 1414.6ms/step | 45.2 samples/s
+Step 4110/10001 | Loss: 3.5235 | LR: 3.24e-04 | 1416.2ms/step | 45.2 samples/s
+Step 4120/10001 | Loss: 3.5575 | LR: 3.23e-04 | 1415.2ms/step | 45.2 samples/s
+Step 4130/10001 | Loss: 3.5274 | LR: 3.22e-04 | 1418.2ms/step | 45.1 samples/s
+Step 4140/10001 | Loss: 3.5132 | LR: 3.21e-04 | 1419.9ms/step | 45.1 samples/s
+Step 4150/10001 | Loss: 3.5222 | LR: 3.20e-04 | 1419.5ms/step | 45.1 samples/s
+Step 4160/10001 | Loss: 3.5117 | LR: 3.20e-04 | 1418.2ms/step | 45.1 samples/s
+Step 4170/10001 | Loss: 3.5378 | LR: 3.19e-04 | 1420.1ms/step | 45.1 samples/s
+Step 4180/10001 | Loss: 3.5513 | LR: 3.18e-04 | 1417.4ms/step | 45.2 samples/s
+Step 4190/10001 | Loss: 3.5047 | LR: 3.17e-04 | 1416.4ms/step | 45.2 samples/s
+Step 4200/10001 | Loss: 3.5451 | LR: 3.17e-04 | 1416.4ms/step | 45.2 samples/s
+Step 4210/10001 | Loss: 3.5340 | LR: 3.16e-04 | 1416.6ms/step | 45.2 samples/s
+Step 4220/10001 | Loss: 3.5230 | LR: 3.15e-04 | 1421.4ms/step | 45.0 samples/s
+Step 4230/10001 | Loss: 3.5255 | LR: 3.14e-04 | 1417.9ms/step | 45.1 samples/s
+Step 4240/10001 | Loss: 3.5287 | LR: 3.14e-04 | 1416.4ms/step | 45.2 samples/s
+Step 4250/10001 | Loss: 3.4970 | LR: 3.13e-04 | 1418.7ms/step | 45.1 samples/s
+Validation at step 4250: Loss = 3.5181, Perplexity = 33.72
+Step 4260/10001 | Loss: 3.5016 | LR: 3.12e-04 | 27693.1ms/step | 2.3 samples/s
+Step 4270/10001 | Loss: 3.5209 | LR: 3.11e-04 | 1413.7ms/step | 45.3 samples/s
+Step 4280/10001 | Loss: 3.4843 | LR: 3.11e-04 | 1414.7ms/step | 45.2 samples/s
+Step 4290/10001 | Loss: 3.5027 | LR: 3.10e-04 | 1417.7ms/step | 45.1 samples/s
+Step 4300/10001 | Loss: 3.4859 | LR: 3.09e-04 | 1422.1ms/step | 45.0 samples/s
+Step 4310/10001 | Loss: 3.5132 | LR: 3.08e-04 | 1417.7ms/step | 45.1 samples/s
+Step 4320/10001 | Loss: 3.5395 | LR: 3.07e-04 | 1417.2ms/step | 45.2 samples/s
+Step 4330/10001 | Loss: 3.5202 | LR: 3.07e-04 | 1417.0ms/step | 45.2 samples/s
+Step 4340/10001 | Loss: 3.5072 | LR: 3.06e-04 | 1417.8ms/step | 45.1 samples/s
+Step 4350/10001 | Loss: 3.5061 | LR: 3.05e-04 | 1415.7ms/step | 45.2 samples/s
+Step 4360/10001 | Loss: 3.4828 | LR: 3.04e-04 | 1414.2ms/step | 45.3 samples/s
+Step 4370/10001 | Loss: 3.5019 | LR: 3.04e-04 | 1417.7ms/step | 45.1 samples/s
+Step 4380/10001 | Loss: 3.5215 | LR: 3.03e-04 | 1419.9ms/step | 45.1 samples/s
+Step 4390/10001 | Loss: 3.5269 | LR: 3.02e-04 | 1414.7ms/step | 45.2 samples/s
+Step 4400/10001 | Loss: 3.5015 | LR: 3.01e-04 | 1496.6ms/step | 42.8 samples/s
+Step 4410/10001 | Loss: 3.5058 | LR: 3.00e-04 | 1415.3ms/step | 45.2 samples/s
+Step 4420/10001 | Loss: 3.4675 | LR: 3.00e-04 | 1422.0ms/step | 45.0 samples/s
+Step 4430/10001 | Loss: 3.5286 | LR: 2.99e-04 | 1421.1ms/step | 45.0 samples/s
+Step 4440/10001 | Loss: 3.4725 | LR: 2.98e-04 | 1421.9ms/step | 45.0 samples/s
+Step 4450/10001 | Loss: 3.5073 | LR: 2.97e-04 | 1422.8ms/step | 45.0 samples/s
+Step 4460/10001 | Loss: 3.5090 | LR: 2.97e-04 | 1419.9ms/step | 45.1 samples/s
+Step 4470/10001 | Loss: 3.4901 | LR: 2.96e-04 | 1416.3ms/step | 45.2 samples/s
+Step 4480/10001 | Loss: 3.4792 | LR: 2.95e-04 | 1420.3ms/step | 45.1 samples/s
+Step 4490/10001 | Loss: 3.5308 | LR: 2.94e-04 | 1421.4ms/step | 45.0 samples/s
+Step 4500/10001 | Loss: 3.4843 | LR: 2.93e-04 | 1419.4ms/step | 45.1 samples/s
+Validation at step 4500: Loss = 3.4973, Perplexity = 33.02
+Step 4510/10001 | Loss: 3.4841 | LR: 2.93e-04 | 27652.2ms/step | 2.3 samples/s
+Step 4520/10001 | Loss: 3.5006 | LR: 2.92e-04 | 1416.7ms/step | 45.2 samples/s
+Step 4530/10001 | Loss: 3.4733 | LR: 2.91e-04 | 1415.5ms/step | 45.2 samples/s
+Step 4540/10001 | Loss: 3.4982 | LR: 2.90e-04 | 1416.4ms/step | 45.2 samples/s
+Step 4550/10001 | Loss: 3.5017 | LR: 2.90e-04 | 1418.1ms/step | 45.1 samples/s
+Step 4560/10001 | Loss: 3.4952 | LR: 2.89e-04 | 1412.6ms/step | 45.3 samples/s
+Step 4570/10001 | Loss: 3.4891 | LR: 2.88e-04 | 1412.7ms/step | 45.3 samples/s
+Step 4580/10001 | Loss: 3.4971 | LR: 2.87e-04 | 1414.5ms/step | 45.2 samples/s
+Step 4590/10001 | Loss: 3.4827 | LR: 2.86e-04 | 1416.3ms/step | 45.2 samples/s
+Step 4600/10001 | Loss: 3.4725 | LR: 2.86e-04 | 1419.1ms/step | 45.1 samples/s
+Step 4610/10001 | Loss: 3.4567 | LR: 2.85e-04 | 1416.1ms/step | 45.2 samples/s
+Step 4620/10001 | Loss: 3.4915 | LR: 2.84e-04 | 1415.8ms/step | 45.2 samples/s
+Step 4630/10001 | Loss: 3.4827 | LR: 2.83e-04 | 1417.1ms/step | 45.2 samples/s
+Step 4640/10001 | Loss: 3.5040 | LR: 2.82e-04 | 1419.0ms/step | 45.1 samples/s
+Step 4650/10001 | Loss: 3.4942 | LR: 2.82e-04 | 1415.5ms/step | 45.2 samples/s
+Step 4660/10001 | Loss: 3.4848 | LR: 2.81e-04 | 1415.1ms/step | 45.2 samples/s
+Step 4670/10001 | Loss: 3.4537 | LR: 2.80e-04 | 1418.4ms/step | 45.1 samples/s
+Step 4680/10001 | Loss: 3.4640 | LR: 2.79e-04 | 1416.6ms/step | 45.2 samples/s
+Step 4690/10001 | Loss: 3.4560 | LR: 2.79e-04 | 1412.7ms/step | 45.3 samples/s
+Step 4700/10001 | Loss: 3.4424 | LR: 2.78e-04 | 1414.1ms/step | 45.3 samples/s
+Step 4710/10001 | Loss: 3.4746 | LR: 2.77e-04 | 1413.7ms/step | 45.3 samples/s
+Step 4720/10001 | Loss: 3.5228 | LR: 2.76e-04 | 1416.6ms/step | 45.2 samples/s
+Step 4730/10001 | Loss: 3.4817 | LR: 2.75e-04 | 1420.0ms/step | 45.1 samples/s
+Step 4740/10001 | Loss: 3.4652 | LR: 2.75e-04 | 1417.3ms/step | 45.2 samples/s
+Step 4750/10001 | Loss: 3.4658 | LR: 2.74e-04 | 1415.1ms/step | 45.2 samples/s
+Validation at step 4750: Loss = 3.4774, Perplexity = 32.37
+Step 4760/10001 | Loss: 3.4952 | LR: 2.73e-04 | 27817.7ms/step | 2.3 samples/s
+Step 4770/10001 | Loss: 3.4865 | LR: 2.72e-04 | 1428.4ms/step | 44.8 samples/s
+Step 4780/10001 | Loss: 3.4916 | LR: 2.71e-04 | 1437.0ms/step | 44.5 samples/s
+Step 4790/10001 | Loss: 3.4759 | LR: 2.71e-04 | 1438.2ms/step | 44.5 samples/s
+Step 4800/10001 | Loss: 3.4666 | LR: 2.70e-04 | 1438.5ms/step | 44.5 samples/s
+Step 4810/10001 | Loss: 3.4660 | LR: 2.69e-04 | 1438.1ms/step | 44.5 samples/s
+Step 4820/10001 | Loss: 3.4642 | LR: 2.68e-04 | 1437.6ms/step | 44.5 samples/s
+Step 4830/10001 | Loss: 3.4484 | LR: 2.67e-04 | 1436.1ms/step | 44.6 samples/s
+Step 4840/10001 | Loss: 3.4997 | LR: 2.67e-04 | 1432.2ms/step | 44.7 samples/s
+Step 4850/10001 | Loss: 3.4767 | LR: 2.66e-04 | 1451.7ms/step | 44.1 samples/s
+Step 4860/10001 | Loss: 3.4790 | LR: 2.65e-04 | 1438.4ms/step | 44.5 samples/s
+Step 4870/10001 | Loss: 3.4924 | LR: 2.64e-04 | 1432.0ms/step | 44.7 samples/s
+Step 4880/10001 | Loss: 3.4496 | LR: 2.64e-04 | 1425.5ms/step | 44.9 samples/s
+Step 4890/10001 | Loss: 3.4741 | LR: 2.63e-04 | 1421.4ms/step | 45.0 samples/s
+Step 4900/10001 | Loss: 3.4655 | LR: 2.62e-04 | 1422.5ms/step | 45.0 samples/s
+Step 4910/10001 | Loss: 3.4711 | LR: 2.61e-04 | 1423.2ms/step | 45.0 samples/s
+Step 4920/10001 | Loss: 3.4372 | LR: 2.60e-04 | 1423.9ms/step | 44.9 samples/s
+Step 4930/10001 | Loss: 3.4321 | LR: 2.60e-04 | 1424.4ms/step | 44.9 samples/s
+Step 4940/10001 | Loss: 3.4617 | LR: 2.59e-04 | 1423.4ms/step | 45.0 samples/s
+Step 4950/10001 | Loss: 3.4998 | LR: 2.58e-04 | 1423.4ms/step | 45.0 samples/s
+Step 4960/10001 | Loss: 3.4745 | LR: 2.57e-04 | 1423.0ms/step | 45.0 samples/s
+Step 4970/10001 | Loss: 3.4476 | LR: 2.56e-04 | 1423.1ms/step | 45.0 samples/s
+Step 4980/10001 | Loss: 3.4641 | LR: 2.56e-04 | 1422.9ms/step | 45.0 samples/s
+Step 4990/10001 | Loss: 3.4676 | LR: 2.55e-04 | 1424.4ms/step | 44.9 samples/s
+Step 5000/10001 | Loss: 3.4600 | LR: 2.54e-04 | 1424.3ms/step | 44.9 samples/s
+Validation at step 5000: Loss = 3.4594, Perplexity = 31.80
+Saved checkpoint to checkpoints/run_20260430_021105/checkpoint-5000
+Step 5010/10001 | Loss: 3.4884 | LR: 2.53e-04 | 28318.6ms/step | 2.3 samples/s
+Step 5020/10001 | Loss: 3.4399 | LR: 2.52e-04 | 1421.9ms/step | 45.0 samples/s
+Step 5030/10001 | Loss: 3.4236 | LR: 2.52e-04 | 1422.6ms/step | 45.0 samples/s
+Step 5040/10001 | Loss: 3.4760 | LR: 2.51e-04 | 1423.6ms/step | 45.0 samples/s
+Step 5050/10001 | Loss: 3.4700 | LR: 2.50e-04 | 1421.0ms/step | 45.0 samples/s
+Step 5060/10001 | Loss: 3.4596 | LR: 2.49e-04 | 1416.7ms/step | 45.2 samples/s
+Step 5070/10001 | Loss: 3.4720 | LR: 2.48e-04 | 1417.3ms/step | 45.2 samples/s
+Step 5080/10001 | Loss: 3.4480 | LR: 2.48e-04 | 1420.9ms/step | 45.0 samples/s
+Step 5090/10001 | Loss: 3.4656 | LR: 2.47e-04 | 1421.1ms/step | 45.0 samples/s
+Step 5100/10001 | Loss: 3.4332 | LR: 2.46e-04 | 1419.0ms/step | 45.1 samples/s
+Step 5110/10001 | Loss: 3.4547 | LR: 2.45e-04 | 1420.6ms/step | 45.1 samples/s
+Step 5120/10001 | Loss: 3.4508 | LR: 2.44e-04 | 1421.6ms/step | 45.0 samples/s
+Step 5130/10001 | Loss: 3.4523 | LR: 2.44e-04 | 1425.8ms/step | 44.9 samples/s
+Step 5140/10001 | Loss: 3.4767 | LR: 2.43e-04 | 1509.0ms/step | 42.4 samples/s
+Step 5150/10001 | Loss: 3.4446 | LR: 2.42e-04 | 1424.4ms/step | 44.9 samples/s
+Step 5160/10001 | Loss: 3.4560 | LR: 2.41e-04 | 1423.6ms/step | 45.0 samples/s
+Step 5170/10001 | Loss: 3.4662 | LR: 2.41e-04 | 1425.6ms/step | 44.9 samples/s
+Step 5180/10001 | Loss: 3.4639 | LR: 2.40e-04 | 1426.9ms/step | 44.9 samples/s
+Step 5190/10001 | Loss: 3.4433 | LR: 2.39e-04 | 1421.7ms/step | 45.0 samples/s
+Step 5200/10001 | Loss: 3.4435 | LR: 2.38e-04 | 1421.3ms/step | 45.0 samples/s
+Step 5210/10001 | Loss: 3.4590 | LR: 2.37e-04 | 1420.5ms/step | 45.1 samples/s
+Step 5220/10001 | Loss: 3.4463 | LR: 2.37e-04 | 1420.9ms/step | 45.0 samples/s
+Step 5230/10001 | Loss: 3.4580 | LR: 2.36e-04 | 1421.7ms/step | 45.0 samples/s
+Step 5240/10001 | Loss: 3.4514 | LR: 2.35e-04 | 1422.1ms/step | 45.0 samples/s
+Step 5250/10001 | Loss: 3.4155 | LR: 2.34e-04 | 1420.8ms/step | 45.0 samples/s
+Validation at step 5250: Loss = 3.4426, Perplexity = 31.27
+Step 5260/10001 | Loss: 3.4322 | LR: 2.33e-04 | 27771.1ms/step | 2.3 samples/s
+Step 5270/10001 | Loss: 3.4416 | LR: 2.33e-04 | 1421.5ms/step | 45.0 samples/s
+Step 5280/10001 | Loss: 3.4264 | LR: 2.32e-04 | 1423.0ms/step | 45.0 samples/s
+Step 5290/10001 | Loss: 3.4228 | LR: 2.31e-04 | 1425.2ms/step | 44.9 samples/s
+Step 5300/10001 | Loss: 3.4789 | LR: 2.30e-04 | 1424.2ms/step | 44.9 samples/s
+Step 5310/10001 | Loss: 3.4330 | LR: 2.29e-04 | 1426.7ms/step | 44.9 samples/s
+Step 5320/10001 | Loss: 3.4148 | LR: 2.29e-04 | 1424.0ms/step | 44.9 samples/s
+Step 5330/10001 | Loss: 3.4208 | LR: 2.28e-04 | 1420.3ms/step | 45.1 samples/s
+Step 5340/10001 | Loss: 3.4212 | LR: 2.27e-04 | 1420.4ms/step | 45.1 samples/s
+Step 5350/10001 | Loss: 3.4507 | LR: 2.26e-04 | 1422.3ms/step | 45.0 samples/s
+Step 5360/10001 | Loss: 3.4322 | LR: 2.25e-04 | 1420.4ms/step | 45.1 samples/s
+Step 5370/10001 | Loss: 3.4401 | LR: 2.25e-04 | 1420.9ms/step | 45.0 samples/s
+Step 5380/10001 | Loss: 3.4215 | LR: 2.24e-04 | 1423.0ms/step | 45.0 samples/s
+Step 5390/10001 | Loss: 3.4213 | LR: 2.23e-04 | 1423.1ms/step | 45.0 samples/s
+Step 5400/10001 | Loss: 3.4707 | LR: 2.22e-04 | 1419.3ms/step | 45.1 samples/s
+Step 5410/10001 | Loss: 3.4225 | LR: 2.22e-04 | 1420.5ms/step | 45.1 samples/s
+Step 5420/10001 | Loss: 3.4377 | LR: 2.21e-04 | 1420.1ms/step | 45.1 samples/s
+Step 5430/10001 | Loss: 3.4454 | LR: 2.20e-04 | 1419.1ms/step | 45.1 samples/s
+Step 5440/10001 | Loss: 3.4503 | LR: 2.19e-04 | 1420.2ms/step | 45.1 samples/s
+Step 5450/10001 | Loss: 3.4326 | LR: 2.18e-04 | 1422.5ms/step | 45.0 samples/s
+Step 5460/10001 | Loss: 3.4480 | LR: 2.18e-04 | 1425.2ms/step | 44.9 samples/s
+Step 5470/10001 | Loss: 3.4656 | LR: 2.17e-04 | 1425.6ms/step | 44.9 samples/s
+Step 5480/10001 | Loss: 3.4447 | LR: 2.16e-04 | 1422.7ms/step | 45.0 samples/s
+Step 5490/10001 | Loss: 3.4185 | LR: 2.15e-04 | 1419.5ms/step | 45.1 samples/s
+Step 5500/10001 | Loss: 3.4188 | LR: 2.14e-04 | 1418.6ms/step | 45.1 samples/s
+Validation at step 5500: Loss = 3.4278, Perplexity = 30.81
+Step 5510/10001 | Loss: 3.4259 | LR: 2.14e-04 | 27804.0ms/step | 2.3 samples/s
+Step 5520/10001 | Loss: 3.4423 | LR: 2.13e-04 | 1427.0ms/step | 44.9 samples/s
+Step 5530/10001 | Loss: 3.4471 | LR: 2.12e-04 | 1426.7ms/step | 44.9 samples/s
+Step 5540/10001 | Loss: 3.4444 | LR: 2.11e-04 | 1422.4ms/step | 45.0 samples/s
+Step 5550/10001 | Loss: 3.4226 | LR: 2.11e-04 | 1422.8ms/step | 45.0 samples/s
+Step 5560/10001 | Loss: 3.4292 | LR: 2.10e-04 | 1422.8ms/step | 45.0 samples/s
+Step 5570/10001 | Loss: 3.3976 | LR: 2.09e-04 | 1422.9ms/step | 45.0 samples/s
+Step 5580/10001 | Loss: 3.4325 | LR: 2.08e-04 | 1424.3ms/step | 44.9 samples/s
+Step 5590/10001 | Loss: 3.4390 | LR: 2.07e-04 | 1425.0ms/step | 44.9 samples/s
+Step 5600/10001 | Loss: 3.4263 | LR: 2.07e-04 | 1424.2ms/step | 44.9 samples/s
+Step 5610/10001 | Loss: 3.4464 | LR: 2.06e-04 | 1425.5ms/step | 44.9 samples/s
+Step 5620/10001 | Loss: 3.4176 | LR: 2.05e-04 | 1424.6ms/step | 44.9 samples/s
+Step 5630/10001 | Loss: 3.4059 | LR: 2.04e-04 | 1417.6ms/step | 45.1 samples/s
+Step 5640/10001 | Loss: 3.3990 | LR: 2.04e-04 | 1420.7ms/step | 45.0 samples/s
+Step 5650/10001 | Loss: 3.4186 | LR: 2.03e-04 | 1421.1ms/step | 45.0 samples/s
+Step 5660/10001 | Loss: 3.4337 | LR: 2.02e-04 | 1420.6ms/step | 45.1 samples/s
+Step 5670/10001 | Loss: 3.4368 | LR: 2.01e-04 | 1421.7ms/step | 45.0 samples/s
+Step 5680/10001 | Loss: 3.4200 | LR: 2.00e-04 | 1423.0ms/step | 45.0 samples/s
+Step 5690/10001 | Loss: 3.4333 | LR: 2.00e-04 | 1422.7ms/step | 45.0 samples/s
+Step 5700/10001 | Loss: 3.4199 | LR: 1.99e-04 | 1425.2ms/step | 44.9 samples/s
+Step 5710/10001 | Loss: 3.4085 | LR: 1.98e-04 | 1423.0ms/step | 45.0 samples/s
+Step 5720/10001 | Loss: 3.3838 | LR: 1.97e-04 | 1423.7ms/step | 45.0 samples/s
+Step 5730/10001 | Loss: 3.3906 | LR: 1.97e-04 | 1421.3ms/step | 45.0 samples/s
+Step 5740/10001 | Loss: 3.4126 | LR: 1.96e-04 | 1422.3ms/step | 45.0 samples/s
+Step 5750/10001 | Loss: 3.4366 | LR: 1.95e-04 | 1421.3ms/step | 45.0 samples/s
+Validation at step 5750: Loss = 3.4141, Perplexity = 30.39
+Step 5760/10001 | Loss: 3.4020 | LR: 1.94e-04 | 27809.7ms/step | 2.3 samples/s
+Step 5770/10001 | Loss: 3.4225 | LR: 1.93e-04 | 1427.2ms/step | 44.8 samples/s
+Step 5780/10001 | Loss: 3.4289 | LR: 1.93e-04 | 1426.5ms/step | 44.9 samples/s
+Step 5790/10001 | Loss: 3.4200 | LR: 1.92e-04 | 1422.7ms/step | 45.0 samples/s
+Step 5800/10001 | Loss: 3.3693 | LR: 1.91e-04 | 1421.7ms/step | 45.0 samples/s
+Step 5810/10001 | Loss: 3.4242 | LR: 1.90e-04 | 1423.1ms/step | 45.0 samples/s
+Step 5820/10001 | Loss: 3.3831 | LR: 1.90e-04 | 1421.5ms/step | 45.0 samples/s
+Step 5830/10001 | Loss: 3.4165 | LR: 1.89e-04 | 1422.5ms/step | 45.0 samples/s
+Step 5840/10001 | Loss: 3.4194 | LR: 1.88e-04 | 1423.9ms/step | 44.9 samples/s
+Step 5850/10001 | Loss: 3.4202 | LR: 1.87e-04 | 1424.1ms/step | 44.9 samples/s
+Step 5860/10001 | Loss: 3.4037 | LR: 1.86e-04 | 1423.5ms/step | 45.0 samples/s
+Step 5870/10001 | Loss: 3.4028 | LR: 1.86e-04 | 1422.5ms/step | 45.0 samples/s
+Step 5880/10001 | Loss: 3.4209 | LR: 1.85e-04 | 1420.5ms/step | 45.1 samples/s
+Step 5890/10001 | Loss: 3.4005 | LR: 1.84e-04 | 1503.5ms/step | 42.6 samples/s
+Step 5900/10001 | Loss: 3.3917 | LR: 1.83e-04 | 1420.8ms/step | 45.0 samples/s
+Step 5910/10001 | Loss: 3.3813 | LR: 1.83e-04 | 1423.7ms/step | 45.0 samples/s
+Step 5920/10001 | Loss: 3.3878 | LR: 1.82e-04 | 1425.4ms/step | 44.9 samples/s
+Step 5930/10001 | Loss: 3.4109 | LR: 1.81e-04 | 1425.8ms/step | 44.9 samples/s
+Step 5940/10001 | Loss: 3.4293 | LR: 1.80e-04 | 1428.5ms/step | 44.8 samples/s
+Step 5950/10001 | Loss: 3.3990 | LR: 1.80e-04 | 1425.6ms/step | 44.9 samples/s
+Step 5960/10001 | Loss: 3.4087 | LR: 1.79e-04 | 1423.3ms/step | 45.0 samples/s
+Step 5970/10001 | Loss: 3.3953 | LR: 1.78e-04 | 1425.4ms/step | 44.9 samples/s
+Step 5980/10001 | Loss: 3.4065 | LR: 1.77e-04 | 1424.4ms/step | 44.9 samples/s
+Step 5990/10001 | Loss: 3.4542 | LR: 1.77e-04 | 1426.1ms/step | 44.9 samples/s
+Step 6000/10001 | Loss: 3.3812 | LR: 1.76e-04 | 1425.3ms/step | 44.9 samples/s
+Validation at step 6000: Loss = 3.4025, Perplexity = 30.04
+Step 6010/10001 | Loss: 3.3926 | LR: 1.75e-04 | 27780.0ms/step | 2.3 samples/s
+Step 6020/10001 | Loss: 3.3771 | LR: 1.74e-04 | 1423.2ms/step | 45.0 samples/s
+Step 6030/10001 | Loss: 3.3961 | LR: 1.74e-04 | 1424.3ms/step | 44.9 samples/s
+Step 6040/10001 | Loss: 3.3841 | LR: 1.73e-04 | 1422.7ms/step | 45.0 samples/s
+Step 6050/10001 | Loss: 3.3941 | LR: 1.72e-04 | 1420.0ms/step | 45.1 samples/s
+Step 6060/10001 | Loss: 3.3976 | LR: 1.71e-04 | 1420.0ms/step | 45.1 samples/s
+Step 6070/10001 | Loss: 3.4228 | LR: 1.71e-04 | 1421.5ms/step | 45.0 samples/s
+Step 6080/10001 | Loss: 3.3849 | LR: 1.70e-04 | 1423.4ms/step | 45.0 samples/s
+Step 6090/10001 | Loss: 3.4185 | LR: 1.69e-04 | 1424.0ms/step | 44.9 samples/s
+Step 6100/10001 | Loss: 3.4406 | LR: 1.68e-04 | 1423.0ms/step | 45.0 samples/s
+Step 6110/10001 | Loss: 3.3770 | LR: 1.68e-04 | 1424.0ms/step | 44.9 samples/s
+Step 6120/10001 | Loss: 3.3638 | LR: 1.67e-04 | 1424.0ms/step | 44.9 samples/s
+Step 6130/10001 | Loss: 3.3872 | LR: 1.66e-04 | 1422.8ms/step | 45.0 samples/s
+Step 6140/10001 | Loss: 3.3962 | LR: 1.65e-04 | 1423.2ms/step | 45.0 samples/s
+Step 6150/10001 | Loss: 3.4194 | LR: 1.65e-04 | 1422.6ms/step | 45.0 samples/s
+Step 6160/10001 | Loss: 3.3824 | LR: 1.64e-04 | 1421.8ms/step | 45.0 samples/s
+Step 6170/10001 | Loss: 3.3913 | LR: 1.63e-04 | 1422.4ms/step | 45.0 samples/s
+Step 6180/10001 | Loss: 3.4003 | LR: 1.62e-04 | 1423.3ms/step | 45.0 samples/s
+Step 6190/10001 | Loss: 3.3943 | LR: 1.62e-04 | 1422.6ms/step | 45.0 samples/s
+Step 6200/10001 | Loss: 3.3880 | LR: 1.61e-04 | 1426.1ms/step | 44.9 samples/s
+Step 6210/10001 | Loss: 3.3873 | LR: 1.60e-04 | 1426.5ms/step | 44.9 samples/s
+Step 6220/10001 | Loss: 3.3659 | LR: 1.59e-04 | 1427.3ms/step | 44.8 samples/s
+Step 6230/10001 | Loss: 3.3746 | LR: 1.59e-04 | 1426.9ms/step | 44.9 samples/s
+Step 6240/10001 | Loss: 3.4168 | LR: 1.58e-04 | 1427.7ms/step | 44.8 samples/s
+Step 6250/10001 | Loss: 3.3947 | LR: 1.57e-04 | 1426.1ms/step | 44.9 samples/s
+Validation at step 6250: Loss = 3.3920, Perplexity = 29.73
+Step 6260/10001 | Loss: 3.4030 | LR: 1.56e-04 | 27799.8ms/step | 2.3 samples/s
+Step 6270/10001 | Loss: 3.3811 | LR: 1.56e-04 | 1424.9ms/step | 44.9 samples/s
+Step 6280/10001 | Loss: 3.3664 | LR: 1.55e-04 | 1417.8ms/step | 45.1 samples/s
+Step 6290/10001 | Loss: 3.3873 | LR: 1.54e-04 | 1420.2ms/step | 45.1 samples/s
+Step 6300/10001 | Loss: 3.4108 | LR: 1.53e-04 | 1423.4ms/step | 45.0 samples/s
+Step 6310/10001 | Loss: 3.3916 | LR: 1.53e-04 | 1422.6ms/step | 45.0 samples/s
+Step 6320/10001 | Loss: 3.3644 | LR: 1.52e-04 | 1423.7ms/step | 45.0 samples/s
+Step 6330/10001 | Loss: 3.4052 | LR: 1.51e-04 | 1421.7ms/step | 45.0 samples/s
+Step 6340/10001 | Loss: 3.3812 | LR: 1.51e-04 | 1424.7ms/step | 44.9 samples/s
+Step 6350/10001 | Loss: 3.3850 | LR: 1.50e-04 | 1426.5ms/step | 44.9 samples/s
+Step 6360/10001 | Loss: 3.3931 | LR: 1.49e-04 | 1423.8ms/step | 44.9 samples/s
+Step 6370/10001 | Loss: 3.3675 | LR: 1.48e-04 | 1420.4ms/step | 45.1 samples/s
+Step 6380/10001 | Loss: 3.3683 | LR: 1.48e-04 | 1421.5ms/step | 45.0 samples/s
+Step 6390/10001 | Loss: 3.3922 | LR: 1.47e-04 | 1421.8ms/step | 45.0 samples/s
+Step 6400/10001 | Loss: 3.3883 | LR: 1.46e-04 | 1423.5ms/step | 45.0 samples/s
+Step 6410/10001 | Loss: 3.4023 | LR: 1.45e-04 | 1424.1ms/step | 44.9 samples/s
+Step 6420/10001 | Loss: 3.3927 | LR: 1.45e-04 | 1422.7ms/step | 45.0 samples/s
+Step 6430/10001 | Loss: 3.3752 | LR: 1.44e-04 | 1423.5ms/step | 45.0 samples/s
+Step 6440/10001 | Loss: 3.3989 | LR: 1.43e-04 | 1423.4ms/step | 45.0 samples/s
+Step 6450/10001 | Loss: 3.3711 | LR: 1.43e-04 | 1423.9ms/step | 44.9 samples/s
+Step 6460/10001 | Loss: 3.3931 | LR: 1.42e-04 | 1424.8ms/step | 44.9 samples/s
+Step 6470/10001 | Loss: 3.3779 | LR: 1.41e-04 | 1425.7ms/step | 44.9 samples/s
+Step 6480/10001 | Loss: 3.4107 | LR: 1.40e-04 | 1427.9ms/step | 44.8 samples/s
+Step 6490/10001 | Loss: 3.3715 | LR: 1.40e-04 | 1428.0ms/step | 44.8 samples/s
+Step 6500/10001 | Loss: 3.3583 | LR: 1.39e-04 | 1426.8ms/step | 44.9 samples/s
+Validation at step 6500: Loss = 3.3824, Perplexity = 29.44
+Step 6510/10001 | Loss: 3.3691 | LR: 1.38e-04 | 27805.5ms/step | 2.3 samples/s
+Step 6520/10001 | Loss: 3.3682 | LR: 1.38e-04 | 1427.4ms/step | 44.8 samples/s
+Step 6530/10001 | Loss: 3.4026 | LR: 1.37e-04 | 1428.1ms/step | 44.8 samples/s
+Step 6540/10001 | Loss: 3.3904 | LR: 1.36e-04 | 1426.0ms/step | 44.9 samples/s
+Step 6550/10001 | Loss: 3.4144 | LR: 1.35e-04 | 1426.3ms/step | 44.9 samples/s
+Step 6560/10001 | Loss: 3.3826 | LR: 1.35e-04 | 1434.3ms/step | 44.6 samples/s
+Step 6570/10001 | Loss: 3.3442 | LR: 1.34e-04 | 1428.1ms/step | 44.8 samples/s
+Step 6580/10001 | Loss: 3.3908 | LR: 1.33e-04 | 1427.2ms/step | 44.8 samples/s
+Step 6590/10001 | Loss: 3.3780 | LR: 1.33e-04 | 1426.1ms/step | 44.9 samples/s
+Step 6600/10001 | Loss: 3.3718 | LR: 1.32e-04 | 1426.3ms/step | 44.9 samples/s
+Step 6610/10001 | Loss: 3.3752 | LR: 1.31e-04 | 1424.7ms/step | 44.9 samples/s
+Step 6620/10001 | Loss: 3.3525 | LR: 1.31e-04 | 1422.9ms/step | 45.0 samples/s
+Step 6630/10001 | Loss: 3.3640 | LR: 1.30e-04 | 1422.6ms/step | 45.0 samples/s
+Step 6640/10001 | Loss: 3.3629 | LR: 1.29e-04 | 1508.1ms/step | 42.4 samples/s
+Step 6650/10001 | Loss: 3.3801 | LR: 1.28e-04 | 1421.8ms/step | 45.0 samples/s
+Step 6660/10001 | Loss: 3.3882 | LR: 1.28e-04 | 1419.1ms/step | 45.1 samples/s
+Step 6670/10001 | Loss: 3.3954 | LR: 1.27e-04 | 1416.6ms/step | 45.2 samples/s
+Step 6680/10001 | Loss: 3.3732 | LR: 1.26e-04 | 1417.4ms/step | 45.2 samples/s
+Step 6690/10001 | Loss: 3.3836 | LR: 1.26e-04 | 1418.9ms/step | 45.1 samples/s
+Step 6700/10001 | Loss: 3.3618 | LR: 1.25e-04 | 1417.3ms/step | 45.2 samples/s
+Step 6710/10001 | Loss: 3.3857 | LR: 1.24e-04 | 1411.5ms/step | 45.3 samples/s
+Step 6720/10001 | Loss: 3.3782 | LR: 1.24e-04 | 1410.0ms/step | 45.4 samples/s
+Step 6730/10001 | Loss: 3.3639 | LR: 1.23e-04 | 1410.5ms/step | 45.4 samples/s
+Step 6740/10001 | Loss: 3.3558 | LR: 1.22e-04 | 1412.2ms/step | 45.3 samples/s
+Step 6750/10001 | Loss: 3.3747 | LR: 1.22e-04 | 1414.4ms/step | 45.2 samples/s
+Validation at step 6750: Loss = 3.3744, Perplexity = 29.21
+Step 6760/10001 | Loss: 3.3740 | LR: 1.21e-04 | 27699.4ms/step | 2.3 samples/s
+Step 6770/10001 | Loss: 3.3704 | LR: 1.20e-04 | 1418.4ms/step | 45.1 samples/s
+Step 6780/10001 | Loss: 3.3886 | LR: 1.20e-04 | 1417.6ms/step | 45.1 samples/s
+Step 6790/10001 | Loss: 3.3682 | LR: 1.19e-04 | 1413.0ms/step | 45.3 samples/s
+Step 6800/10001 | Loss: 3.3478 | LR: 1.18e-04 | 1416.4ms/step | 45.2 samples/s
+Step 6810/10001 | Loss: 3.3679 | LR: 1.18e-04 | 1416.7ms/step | 45.2 samples/s
+Step 6820/10001 | Loss: 3.3754 | LR: 1.17e-04 | 1420.1ms/step | 45.1 samples/s
+Step 6830/10001 | Loss: 3.3488 | LR: 1.16e-04 | 1417.5ms/step | 45.1 samples/s
+Step 6840/10001 | Loss: 3.3801 | LR: 1.16e-04 | 1416.2ms/step | 45.2 samples/s
+Step 6850/10001 | Loss: 3.3600 | LR: 1.15e-04 | 1414.0ms/step | 45.3 samples/s
+Step 6860/10001 | Loss: 3.3836 | LR: 1.14e-04 | 1415.8ms/step | 45.2 samples/s
+Step 6870/10001 | Loss: 3.3522 | LR: 1.14e-04 | 1417.3ms/step | 45.2 samples/s
+Step 6880/10001 | Loss: 3.3594 | LR: 1.13e-04 | 1411.0ms/step | 45.4 samples/s
+Step 6890/10001 | Loss: 3.3932 | LR: 1.12e-04 | 1406.5ms/step | 45.5 samples/s
+Step 6900/10001 | Loss: 3.3833 | LR: 1.12e-04 | 1408.5ms/step | 45.4 samples/s
+Step 6910/10001 | Loss: 3.3609 | LR: 1.11e-04 | 1411.8ms/step | 45.3 samples/s
+Step 6920/10001 | Loss: 3.3677 | LR: 1.10e-04 | 1411.9ms/step | 45.3 samples/s
+Step 6930/10001 | Loss: 3.3563 | LR: 1.10e-04 | 1412.3ms/step | 45.3 samples/s
+Step 6940/10001 | Loss: 3.3620 | LR: 1.09e-04 | 1414.9ms/step | 45.2 samples/s
+Step 6950/10001 | Loss: 3.3782 | LR: 1.08e-04 | 1415.6ms/step | 45.2 samples/s
+Step 6960/10001 | Loss: 3.3840 | LR: 1.08e-04 | 1413.2ms/step | 45.3 samples/s
+Step 6970/10001 | Loss: 3.3532 | LR: 1.07e-04 | 1413.4ms/step | 45.3 samples/s
+Step 6980/10001 | Loss: 3.3818 | LR: 1.06e-04 | 1417.6ms/step | 45.1 samples/s
+Step 6990/10001 | Loss: 3.3690 | LR: 1.06e-04 | 1421.2ms/step | 45.0 samples/s
+Step 7000/10001 | Loss: 3.3412 | LR: 1.05e-04 | 1416.9ms/step | 45.2 samples/s
+Validation at step 7000: Loss = 3.3679, Perplexity = 29.02
+Step 7010/10001 | Loss: 3.3242 | LR: 1.04e-04 | 27684.5ms/step | 2.3 samples/s
+Step 7020/10001 | Loss: 3.3674 | LR: 1.04e-04 | 1421.6ms/step | 45.0 samples/s
+Step 7030/10001 | Loss: 3.3158 | LR: 1.03e-04 | 1419.4ms/step | 45.1 samples/s
+Step 7040/10001 | Loss: 3.3848 | LR: 1.02e-04 | 1418.9ms/step | 45.1 samples/s
+Step 7050/10001 | Loss: 3.3652 | LR: 1.02e-04 | 1420.3ms/step | 45.1 samples/s
+Step 7060/10001 | Loss: 3.3682 | LR: 1.01e-04 | 1419.3ms/step | 45.1 samples/s
+Step 7070/10001 | Loss: 3.3641 | LR: 1.01e-04 | 1418.6ms/step | 45.1 samples/s
+Step 7080/10001 | Loss: 3.3356 | LR: 9.99e-05 | 1419.6ms/step | 45.1 samples/s
+Step 7090/10001 | Loss: 3.3558 | LR: 9.93e-05 | 1418.0ms/step | 45.1 samples/s
+Step 7100/10001 | Loss: 3.3530 | LR: 9.86e-05 | 1411.8ms/step | 45.3 samples/s
+Step 7110/10001 | Loss: 3.3406 | LR: 9.80e-05 | 1410.0ms/step | 45.4 samples/s
+Step 7120/10001 | Loss: 3.3486 | LR: 9.74e-05 | 1413.9ms/step | 45.3 samples/s
+Step 7130/10001 | Loss: 3.3719 | LR: 9.68e-05 | 1413.3ms/step | 45.3 samples/s
+Step 7140/10001 | Loss: 3.3678 | LR: 9.61e-05 | 1411.1ms/step | 45.4 samples/s
+Step 7150/10001 | Loss: 3.3750 | LR: 9.55e-05 | 1413.8ms/step | 45.3 samples/s
+Step 7160/10001 | Loss: 3.3879 | LR: 9.49e-05 | 1414.7ms/step | 45.2 samples/s
+Step 7170/10001 | Loss: 3.3477 | LR: 9.43e-05 | 1415.0ms/step | 45.2 samples/s
+Step 7180/10001 | Loss: 3.3490 | LR: 9.36e-05 | 1414.3ms/step | 45.3 samples/s
+Step 7190/10001 | Loss: 3.3646 | LR: 9.30e-05 | 1415.3ms/step | 45.2 samples/s
+Step 7200/10001 | Loss: 3.3673 | LR: 9.24e-05 | 1414.7ms/step | 45.2 samples/s
+Step 7210/10001 | Loss: 3.3745 | LR: 9.18e-05 | 1413.4ms/step | 45.3 samples/s
+Step 7220/10001 | Loss: 3.3548 | LR: 9.12e-05 | 1410.3ms/step | 45.4 samples/s
+Step 7230/10001 | Loss: 3.3650 | LR: 9.06e-05 | 1412.0ms/step | 45.3 samples/s
+Step 7240/10001 | Loss: 3.3364 | LR: 9.00e-05 | 1411.9ms/step | 45.3 samples/s
+Step 7250/10001 | Loss: 3.3735 | LR: 8.93e-05 | 1414.3ms/step | 45.3 samples/s
+Validation at step 7250: Loss = 3.3630, Perplexity = 28.87
+Step 7260/10001 | Loss: 3.3301 | LR: 8.87e-05 | 27670.5ms/step | 2.3 samples/s
+Step 7270/10001 | Loss: 3.3766 | LR: 8.81e-05 | 1412.2ms/step | 45.3 samples/s
+Step 7280/10001 | Loss: 3.3814 | LR: 8.75e-05 | 1410.7ms/step | 45.4 samples/s
+Step 7290/10001 | Loss: 3.3588 | LR: 8.69e-05 | 1418.9ms/step | 45.1 samples/s
+Step 7300/10001 | Loss: 3.3630 | LR: 8.63e-05 | 1420.0ms/step | 45.1 samples/s
+Step 7310/10001 | Loss: 3.3498 | LR: 8.57e-05 | 1417.5ms/step | 45.2 samples/s
+Step 7320/10001 | Loss: 3.3569 | LR: 8.51e-05 | 1415.6ms/step | 45.2 samples/s
+Step 7330/10001 | Loss: 3.3734 | LR: 8.45e-05 | 1418.5ms/step | 45.1 samples/s
+Step 7340/10001 | Loss: 3.3349 | LR: 8.39e-05 | 1415.6ms/step | 45.2 samples/s
+Step 7350/10001 | Loss: 3.3691 | LR: 8.34e-05 | 1414.3ms/step | 45.3 samples/s
+Step 7360/10001 | Loss: 3.3729 | LR: 8.28e-05 | 1413.4ms/step | 45.3 samples/s
+Step 7370/10001 | Loss: 3.3664 | LR: 8.22e-05 | 1412.7ms/step | 45.3 samples/s
+Step 7380/10001 | Loss: 3.3783 | LR: 8.16e-05 | 1413.5ms/step | 45.3 samples/s
+Step 7390/10001 | Loss: 3.3501 | LR: 8.10e-05 | 1498.9ms/step | 42.7 samples/s
+Step 7400/10001 | Loss: 3.3580 | LR: 8.04e-05 | 1419.5ms/step | 45.1 samples/s
+Step 7410/10001 | Loss: 3.3764 | LR: 7.98e-05 | 1414.5ms/step | 45.2 samples/s
+Step 7420/10001 | Loss: 3.3399 | LR: 7.93e-05 | 1415.7ms/step | 45.2 samples/s
+Step 7430/10001 | Loss: 3.3654 | LR: 7.87e-05 | 1416.9ms/step | 45.2 samples/s
+Step 7440/10001 | Loss: 3.3426 | LR: 7.81e-05 | 1418.7ms/step | 45.1 samples/s
+Step 7450/10001 | Loss: 3.3662 | LR: 7.75e-05 | 1410.3ms/step | 45.4 samples/s
+Step 7460/10001 | Loss: 3.3554 | LR: 7.69e-05 | 1408.2ms/step | 45.4 samples/s
+Step 7470/10001 | Loss: 3.3349 | LR: 7.64e-05 | 1409.9ms/step | 45.4 samples/s
+Step 7480/10001 | Loss: 3.3492 | LR: 7.58e-05 | 1413.9ms/step | 45.3 samples/s
+Step 7490/10001 | Loss: 3.3399 | LR: 7.52e-05 | 1412.7ms/step | 45.3 samples/s
+Step 7500/10001 | Loss: 3.3838 | LR: 7.47e-05 | 1409.4ms/step | 45.4 samples/s
+Validation at step 7500: Loss = 3.3588, Perplexity = 28.75
+Saved checkpoint to checkpoints/run_20260430_021105/checkpoint-7500
+Step 7510/10001 | Loss: 3.3758 | LR: 7.41e-05 | 28128.6ms/step | 2.3 samples/s
+Step 7520/10001 | Loss: 3.3264 | LR: 7.35e-05 | 1410.7ms/step | 45.4 samples/s
+Step 7530/10001 | Loss: 3.3752 | LR: 7.30e-05 | 1413.6ms/step | 45.3 samples/s
+Step 7540/10001 | Loss: 3.3576 | LR: 7.24e-05 | 1417.0ms/step | 45.2 samples/s
+Step 7550/10001 | Loss: 3.3418 | LR: 7.19e-05 | 1416.8ms/step | 45.2 samples/s
+Step 7560/10001 | Loss: 3.3451 | LR: 7.13e-05 | 1417.1ms/step | 45.2 samples/s
+Step 7570/10001 | Loss: 3.3446 | LR: 7.08e-05 | 1412.8ms/step | 45.3 samples/s
+Step 7580/10001 | Loss: 3.3298 | LR: 7.02e-05 | 1411.9ms/step | 45.3 samples/s
+Step 7590/10001 | Loss: 3.3624 | LR: 6.97e-05 | 1415.1ms/step | 45.2 samples/s
+Step 7600/10001 | Loss: 3.3742 | LR: 6.91e-05 | 1415.5ms/step | 45.2 samples/s
+Step 7610/10001 | Loss: 3.3288 | LR: 6.86e-05 | 1414.3ms/step | 45.3 samples/s
+Step 7620/10001 | Loss: 3.3889 | LR: 6.80e-05 | 1414.6ms/step | 45.2 samples/s
+Step 7630/10001 | Loss: 3.3562 | LR: 6.75e-05 | 1412.5ms/step | 45.3 samples/s
+Step 7640/10001 | Loss: 3.3461 | LR: 6.69e-05 | 1410.8ms/step | 45.4 samples/s
+Step 7650/10001 | Loss: 3.3660 | LR: 6.64e-05 | 1410.9ms/step | 45.4 samples/s
+Step 7660/10001 | Loss: 3.3459 | LR: 6.59e-05 | 1411.0ms/step | 45.4 samples/s
+Step 7670/10001 | Loss: 3.3459 | LR: 6.53e-05 | 1412.0ms/step | 45.3 samples/s
+Step 7680/10001 | Loss: 3.3688 | LR: 6.48e-05 | 1419.8ms/step | 45.1 samples/s
+Step 7690/10001 | Loss: 3.3484 | LR: 6.43e-05 | 1420.4ms/step | 45.1 samples/s
+Step 7700/10001 | Loss: 3.3354 | LR: 6.37e-05 | 1417.3ms/step | 45.2 samples/s
+Step 7710/10001 | Loss: 3.3552 | LR: 6.32e-05 | 1417.5ms/step | 45.2 samples/s
+Step 7720/10001 | Loss: 3.3535 | LR: 6.27e-05 | 1416.0ms/step | 45.2 samples/s
+Step 7730/10001 | Loss: 3.3538 | LR: 6.21e-05 | 1410.1ms/step | 45.4 samples/s
+Step 7740/10001 | Loss: 3.3404 | LR: 6.16e-05 | 1409.2ms/step | 45.4 samples/s
+Step 7750/10001 | Loss: 3.3411 | LR: 6.11e-05 | 1412.0ms/step | 45.3 samples/s
+Validation at step 7750: Loss = 3.3558, Perplexity = 28.67
+Step 7760/10001 | Loss: 3.3782 | LR: 6.06e-05 | 27644.2ms/step | 2.3 samples/s
+Step 7770/10001 | Loss: 3.3540 | LR: 6.01e-05 | 1419.1ms/step | 45.1 samples/s
+Step 7780/10001 | Loss: 3.3376 | LR: 5.96e-05 | 1422.4ms/step | 45.0 samples/s
+Step 7790/10001 | Loss: 3.3475 | LR: 5.90e-05 | 1417.6ms/step | 45.1 samples/s
+Step 7800/10001 | Loss: 3.3322 | LR: 5.85e-05 | 1415.6ms/step | 45.2 samples/s
+Step 7810/10001 | Loss: 3.3667 | LR: 5.80e-05 | 1416.0ms/step | 45.2 samples/s
+Step 7820/10001 | Loss: 3.3606 | LR: 5.75e-05 | 1411.2ms/step | 45.4 samples/s
+Step 7830/10001 | Loss: 3.3359 | LR: 5.70e-05 | 1409.6ms/step | 45.4 samples/s
+Step 7840/10001 | Loss: 3.3549 | LR: 5.65e-05 | 1411.4ms/step | 45.3 samples/s
+Step 7850/10001 | Loss: 3.3351 | LR: 5.60e-05 | 1413.4ms/step | 45.3 samples/s
+Step 7860/10001 | Loss: 3.3364 | LR: 5.55e-05 | 1410.4ms/step | 45.4 samples/s
+Step 7870/10001 | Loss: 3.3446 | LR: 5.50e-05 | 1410.5ms/step | 45.4 samples/s
+Step 7880/10001 | Loss: 3.3415 | LR: 5.45e-05 | 1416.1ms/step | 45.2 samples/s
+Step 7890/10001 | Loss: 3.3160 | LR: 5.40e-05 | 1419.4ms/step | 45.1 samples/s
+Step 7900/10001 | Loss: 3.3737 | LR: 5.35e-05 | 1417.7ms/step | 45.1 samples/s
+Step 7910/10001 | Loss: 3.3266 | LR: 5.30e-05 | 1410.7ms/step | 45.4 samples/s
+Step 7920/10001 | Loss: 3.3398 | LR: 5.25e-05 | 1406.7ms/step | 45.5 samples/s
+Step 7930/10001 | Loss: 3.3667 | LR: 5.21e-05 | 1405.1ms/step | 45.5 samples/s
+Step 7940/10001 | Loss: 3.3564 | LR: 5.16e-05 | 1407.2ms/step | 45.5 samples/s
+Step 7950/10001 | Loss: 3.3529 | LR: 5.11e-05 | 1410.4ms/step | 45.4 samples/s
+Step 7960/10001 | Loss: 3.3359 | LR: 5.06e-05 | 1414.4ms/step | 45.2 samples/s
+Step 7970/10001 | Loss: 3.3443 | LR: 5.01e-05 | 1414.3ms/step | 45.3 samples/s
+Step 7980/10001 | Loss: 3.3538 | LR: 4.97e-05 | 1415.7ms/step | 45.2 samples/s
+Step 7990/10001 | Loss: 3.3550 | LR: 4.92e-05 | 1416.2ms/step | 45.2 samples/s
+Step 8000/10001 | Loss: 3.3544 | LR: 4.87e-05 | 1416.8ms/step | 45.2 samples/s
+Validation at step 8000: Loss = 3.3537, Perplexity = 28.61
+Step 8010/10001 | Loss: 3.3388 | LR: 4.83e-05 | 27654.8ms/step | 2.3 samples/s
+Step 8020/10001 | Loss: 3.3460 | LR: 4.78e-05 | 1415.7ms/step | 45.2 samples/s
+Step 8030/10001 | Loss: 3.3551 | LR: 4.73e-05 | 1412.6ms/step | 45.3 samples/s
+Step 8040/10001 | Loss: 3.3216 | LR: 4.69e-05 | 1409.9ms/step | 45.4 samples/s
+Step 8050/10001 | Loss: 3.3617 | LR: 4.64e-05 | 1409.6ms/step | 45.4 samples/s
+Step 8060/10001 | Loss: 3.3467 | LR: 4.59e-05 | 1414.3ms/step | 45.3 samples/s
+Step 8070/10001 | Loss: 3.3518 | LR: 4.55e-05 | 1418.3ms/step | 45.1 samples/s
+Step 8080/10001 | Loss: 3.3413 | LR: 4.50e-05 | 1420.6ms/step | 45.1 samples/s
+Step 8090/10001 | Loss: 3.3172 | LR: 4.46e-05 | 1422.8ms/step | 45.0 samples/s
+Step 8100/10001 | Loss: 3.3681 | LR: 4.41e-05 | 1422.4ms/step | 45.0 samples/s
+Step 8110/10001 | Loss: 3.3394 | LR: 4.37e-05 | 1417.6ms/step | 45.1 samples/s
+Step 8120/10001 | Loss: 3.3724 | LR: 4.32e-05 | 1411.2ms/step | 45.4 samples/s
+Step 8130/10001 | Loss: 3.3374 | LR: 4.28e-05 | 1407.9ms/step | 45.5 samples/s
+Step 8140/10001 | Loss: 3.3435 | LR: 4.23e-05 | 1494.4ms/step | 42.8 samples/s
+Step 8150/10001 | Loss: 3.3477 | LR: 4.19e-05 | 1412.8ms/step | 45.3 samples/s
+Step 8160/10001 | Loss: 3.3793 | LR: 4.15e-05 | 1419.1ms/step | 45.1 samples/s
+Step 8170/10001 | Loss: 3.3537 | LR: 4.10e-05 | 1427.0ms/step | 44.9 samples/s
+Step 8180/10001 | Loss: 3.3448 | LR: 4.06e-05 | 1422.8ms/step | 45.0 samples/s
+Step 8190/10001 | Loss: 3.3724 | LR: 4.02e-05 | 1425.7ms/step | 44.9 samples/s
+Step 8200/10001 | Loss: 3.3571 | LR: 3.97e-05 | 1418.4ms/step | 45.1 samples/s
+Step 8210/10001 | Loss: 3.3455 | LR: 3.93e-05 | 1415.1ms/step | 45.2 samples/s
+Step 8220/10001 | Loss: 3.3424 | LR: 3.89e-05 | 1409.0ms/step | 45.4 samples/s
+Step 8230/10001 | Loss: 3.3384 | LR: 3.84e-05 | 1409.0ms/step | 45.4 samples/s
+Step 8240/10001 | Loss: 3.3453 | LR: 3.80e-05 | 1411.1ms/step | 45.4 samples/s
+Step 8250/10001 | Loss: 3.3590 | LR: 3.76e-05 | 1416.2ms/step | 45.2 samples/s
+Validation at step 8250: Loss = 3.3522, Perplexity = 28.56
+Step 8260/10001 | Loss: 3.3191 | LR: 3.72e-05 | 27685.5ms/step | 2.3 samples/s
+Step 8270/10001 | Loss: 3.3278 | LR: 3.68e-05 | 1417.5ms/step | 45.2 samples/s
+Step 8280/10001 | Loss: 3.3532 | LR: 3.64e-05 | 1415.7ms/step | 45.2 samples/s
+Step 8290/10001 | Loss: 3.3215 | LR: 3.59e-05 | 1414.2ms/step | 45.3 samples/s
+Step 8300/10001 | Loss: 3.3672 | LR: 3.55e-05 | 1414.4ms/step | 45.2 samples/s
+Step 8310/10001 | Loss: 3.3283 | LR: 3.51e-05 | 1416.8ms/step | 45.2 samples/s
+Step 8320/10001 | Loss: 3.3523 | LR: 3.47e-05 | 1417.9ms/step | 45.1 samples/s
+Step 8330/10001 | Loss: 3.3450 | LR: 3.43e-05 | 1418.7ms/step | 45.1 samples/s
+Step 8340/10001 | Loss: 3.3673 | LR: 3.39e-05 | 1421.2ms/step | 45.0 samples/s
+Step 8350/10001 | Loss: 3.3675 | LR: 3.35e-05 | 1420.9ms/step | 45.0 samples/s
+Step 8360/10001 | Loss: 3.3670 | LR: 3.31e-05 | 1415.9ms/step | 45.2 samples/s
+Step 8370/10001 | Loss: 3.3542 | LR: 3.27e-05 | 1414.3ms/step | 45.3 samples/s
+Step 8380/10001 | Loss: 3.3358 | LR: 3.23e-05 | 1415.6ms/step | 45.2 samples/s
+Step 8390/10001 | Loss: 3.3521 | LR: 3.20e-05 | 1415.7ms/step | 45.2 samples/s
+Step 8400/10001 | Loss: 3.3503 | LR: 3.16e-05 | 1411.3ms/step | 45.3 samples/s
+Step 8410/10001 | Loss: 3.3720 | LR: 3.12e-05 | 1416.9ms/step | 45.2 samples/s
+Step 8420/10001 | Loss: 3.3125 | LR: 3.08e-05 | 1412.7ms/step | 45.3 samples/s
+Step 8430/10001 | Loss: 3.3633 | LR: 3.04e-05 | 1408.5ms/step | 45.4 samples/s
+Step 8440/10001 | Loss: 3.3544 | LR: 3.00e-05 | 1411.1ms/step | 45.4 samples/s
+Step 8450/10001 | Loss: 3.3716 | LR: 2.97e-05 | 1415.9ms/step | 45.2 samples/s
+Step 8460/10001 | Loss: 3.3663 | LR: 2.93e-05 | 1417.7ms/step | 45.1 samples/s
+Step 8470/10001 | Loss: 3.3473 | LR: 2.89e-05 | 1411.2ms/step | 45.4 samples/s
+Step 8480/10001 | Loss: 3.3538 | LR: 2.86e-05 | 1412.9ms/step | 45.3 samples/s
+Step 8490/10001 | Loss: 3.3529 | LR: 2.82e-05 | 1417.7ms/step | 45.1 samples/s
+Step 8500/10001 | Loss: 3.3746 | LR: 2.78e-05 | 1422.2ms/step | 45.0 samples/s
+Validation at step 8500: Loss = 3.3513, Perplexity = 28.54
+Step 8510/10001 | Loss: 3.3609 | LR: 2.75e-05 | 27708.8ms/step | 2.3 samples/s
+Step 8520/10001 | Loss: 3.3632 | LR: 2.71e-05 | 1427.3ms/step | 44.8 samples/s
+Step 8530/10001 | Loss: 3.3626 | LR: 2.67e-05 | 1424.7ms/step | 44.9 samples/s
+Step 8540/10001 | Loss: 3.3309 | LR: 2.64e-05 | 1419.3ms/step | 45.1 samples/s
+Step 8550/10001 | Loss: 3.3546 | LR: 2.60e-05 | 1414.9ms/step | 45.2 samples/s
+Step 8560/10001 | Loss: 3.3395 | LR: 2.57e-05 | 1410.8ms/step | 45.4 samples/s
+Step 8570/10001 | Loss: 3.3403 | LR: 2.53e-05 | 1411.3ms/step | 45.3 samples/s
+Step 8580/10001 | Loss: 3.3184 | LR: 2.50e-05 | 1410.4ms/step | 45.4 samples/s
+Step 8590/10001 | Loss: 3.3443 | LR: 2.46e-05 | 1415.8ms/step | 45.2 samples/s
+Step 8600/10001 | Loss: 3.3408 | LR: 2.43e-05 | 1419.6ms/step | 45.1 samples/s
+Step 8610/10001 | Loss: 3.3682 | LR: 2.40e-05 | 1420.4ms/step | 45.1 samples/s
+Step 8620/10001 | Loss: 3.3441 | LR: 2.36e-05 | 1413.2ms/step | 45.3 samples/s
+Step 8630/10001 | Loss: 3.3384 | LR: 2.33e-05 | 1411.2ms/step | 45.4 samples/s
+Step 8640/10001 | Loss: 3.3534 | LR: 2.30e-05 | 1412.7ms/step | 45.3 samples/s
+Step 8650/10001 | Loss: 3.3550 | LR: 2.26e-05 | 1415.6ms/step | 45.2 samples/s
+Step 8660/10001 | Loss: 3.3645 | LR: 2.23e-05 | 1415.8ms/step | 45.2 samples/s
+Step 8670/10001 | Loss: 3.3317 | LR: 2.20e-05 | 1415.6ms/step | 45.2 samples/s
+Step 8680/10001 | Loss: 3.3536 | LR: 2.16e-05 | 1412.4ms/step | 45.3 samples/s
+Step 8690/10001 | Loss: 3.3744 | LR: 2.13e-05 | 1418.3ms/step | 45.1 samples/s
+Step 8700/10001 | Loss: 3.3193 | LR: 2.10e-05 | 1420.2ms/step | 45.1 samples/s
+Step 8710/10001 | Loss: 3.3346 | LR: 2.07e-05 | 1416.7ms/step | 45.2 samples/s
+Step 8720/10001 | Loss: 3.3493 | LR: 2.04e-05 | 1416.9ms/step | 45.2 samples/s
+Step 8730/10001 | Loss: 3.3549 | LR: 2.01e-05 | 1415.5ms/step | 45.2 samples/s
+Step 8740/10001 | Loss: 3.3438 | LR: 1.97e-05 | 1417.6ms/step | 45.1 samples/s
+Step 8750/10001 | Loss: 3.3470 | LR: 1.94e-05 | 1413.9ms/step | 45.3 samples/s
+Validation at step 8750: Loss = 3.3508, Perplexity = 28.53
+Step 8760/10001 | Loss: 3.3622 | LR: 1.91e-05 | 27699.6ms/step | 2.3 samples/s
+Step 8770/10001 | Loss: 3.3696 | LR: 1.88e-05 | 1416.8ms/step | 45.2 samples/s
+Step 8780/10001 | Loss: 3.3503 | LR: 1.85e-05 | 1416.8ms/step | 45.2 samples/s
+Step 8790/10001 | Loss: 3.3641 | LR: 1.82e-05 | 1416.4ms/step | 45.2 samples/s
+Step 8800/10001 | Loss: 3.3631 | LR: 1.79e-05 | 1417.7ms/step | 45.1 samples/s
+Step 8810/10001 | Loss: 3.3321 | LR: 1.76e-05 | 1420.1ms/step | 45.1 samples/s
+Step 8820/10001 | Loss: 3.3392 | LR: 1.73e-05 | 1416.7ms/step | 45.2 samples/s
+Step 8830/10001 | Loss: 3.3431 | LR: 1.71e-05 | 1415.5ms/step | 45.2 samples/s
+Step 8840/10001 | Loss: 3.3730 | LR: 1.68e-05 | 1419.5ms/step | 45.1 samples/s
+Step 8850/10001 | Loss: 3.3590 | LR: 1.65e-05 | 1417.5ms/step | 45.1 samples/s
+Step 8860/10001 | Loss: 3.3360 | LR: 1.62e-05 | 1413.9ms/step | 45.3 samples/s
+Step 8870/10001 | Loss: 3.3317 | LR: 1.59e-05 | 1418.5ms/step | 45.1 samples/s
+Step 8880/10001 | Loss: 3.3385 | LR: 1.56e-05 | 1510.0ms/step | 42.4 samples/s
+Step 8890/10001 | Loss: 3.3598 | LR: 1.54e-05 | 1413.4ms/step | 45.3 samples/s
+Step 8900/10001 | Loss: 3.3358 | LR: 1.51e-05 | 1411.2ms/step | 45.4 samples/s
+Step 8910/10001 | Loss: 3.3538 | LR: 1.48e-05 | 1415.1ms/step | 45.2 samples/s
+Step 8920/10001 | Loss: 3.3431 | LR: 1.46e-05 | 1417.7ms/step | 45.1 samples/s
+Step 8930/10001 | Loss: 3.3528 | LR: 1.43e-05 | 1419.9ms/step | 45.1 samples/s
+Step 8940/10001 | Loss: 3.3301 | LR: 1.40e-05 | 1418.8ms/step | 45.1 samples/s
+Step 8950/10001 | Loss: 3.3472 | LR: 1.38e-05 | 1417.6ms/step | 45.1 samples/s
+Step 8960/10001 | Loss: 3.3441 | LR: 1.35e-05 | 1411.6ms/step | 45.3 samples/s
+Step 8970/10001 | Loss: 3.3455 | LR: 1.33e-05 | 1414.6ms/step | 45.2 samples/s
+Step 8980/10001 | Loss: 3.3370 | LR: 1.30e-05 | 1418.4ms/step | 45.1 samples/s
+Step 8990/10001 | Loss: 3.3699 | LR: 1.28e-05 | 1418.1ms/step | 45.1 samples/s
+Step 9000/10001 | Loss: 3.3690 | LR: 1.25e-05 | 1414.2ms/step | 45.3 samples/s
+Validation at step 9000: Loss = 3.3507, Perplexity = 28.52
+Step 9010/10001 | Loss: 3.3408 | LR: 1.23e-05 | 27668.4ms/step | 2.3 samples/s
+Step 9020/10001 | Loss: 3.3702 | LR: 1.20e-05 | 1408.1ms/step | 45.5 samples/s
+Step 9030/10001 | Loss: 3.3613 | LR: 1.18e-05 | 1407.3ms/step | 45.5 samples/s
+Step 9040/10001 | Loss: 3.3415 | LR: 1.15e-05 | 1411.5ms/step | 45.3 samples/s
+Step 9050/10001 | Loss: 3.3524 | LR: 1.13e-05 | 1418.7ms/step | 45.1 samples/s
+Step 9060/10001 | Loss: 3.3508 | LR: 1.11e-05 | 1436.9ms/step | 44.5 samples/s
+Step 9070/10001 | Loss: 3.3564 | LR: 1.08e-05 | 1428.8ms/step | 44.8 samples/s
+Step 9080/10001 | Loss: 3.3639 | LR: 1.06e-05 | 1416.4ms/step | 45.2 samples/s
+Step 9090/10001 | Loss: 3.3429 | LR: 1.04e-05 | 1413.4ms/step | 45.3 samples/s
+Step 9100/10001 | Loss: 3.3453 | LR: 1.01e-05 | 1419.3ms/step | 45.1 samples/s
+Step 9110/10001 | Loss: 3.3544 | LR: 9.92e-06 | 1416.0ms/step | 45.2 samples/s
+Step 9120/10001 | Loss: 3.3365 | LR: 9.70e-06 | 1411.0ms/step | 45.4 samples/s
+Step 9130/10001 | Loss: 3.3478 | LR: 9.49e-06 | 1414.9ms/step | 45.2 samples/s
+Step 9140/10001 | Loss: 3.3607 | LR: 9.27e-06 | 1417.2ms/step | 45.2 samples/s
+Step 9150/10001 | Loss: 3.3165 | LR: 9.06e-06 | 1420.6ms/step | 45.1 samples/s
+Step 9160/10001 | Loss: 3.3301 | LR: 8.85e-06 | 1424.3ms/step | 44.9 samples/s
+Step 9170/10001 | Loss: 3.3348 | LR: 8.64e-06 | 1419.6ms/step | 45.1 samples/s
+Step 9180/10001 | Loss: 3.3411 | LR: 8.43e-06 | 1412.7ms/step | 45.3 samples/s
+Step 9190/10001 | Loss: 3.3601 | LR: 8.23e-06 | 1408.9ms/step | 45.4 samples/s
+Step 9200/10001 | Loss: 3.3277 | LR: 8.03e-06 | 1411.3ms/step | 45.3 samples/s
+Step 9210/10001 | Loss: 3.3592 | LR: 7.83e-06 | 1412.7ms/step | 45.3 samples/s
+Step 9220/10001 | Loss: 3.3789 | LR: 7.64e-06 | 1414.0ms/step | 45.3 samples/s
+Step 9230/10001 | Loss: 3.3329 | LR: 7.44e-06 | 1410.7ms/step | 45.4 samples/s
+Step 9240/10001 | Loss: 3.3594 | LR: 7.25e-06 | 1411.5ms/step | 45.3 samples/s
+Step 9250/10001 | Loss: 3.3634 | LR: 7.06e-06 | 1414.4ms/step | 45.3 samples/s
+Validation at step 9250: Loss = 3.3506, Perplexity = 28.52
+Step 9260/10001 | Loss: 3.3694 | LR: 6.88e-06 | 27643.3ms/step | 2.3 samples/s
+Step 9270/10001 | Loss: 3.3413 | LR: 6.69e-06 | 1418.2ms/step | 45.1 samples/s
+Step 9280/10001 | Loss: 3.3646 | LR: 6.51e-06 | 1413.0ms/step | 45.3 samples/s
+Step 9290/10001 | Loss: 3.3401 | LR: 6.34e-06 | 1410.6ms/step | 45.4 samples/s
+Step 9300/10001 | Loss: 3.3634 | LR: 6.16e-06 | 1413.6ms/step | 45.3 samples/s
+Step 9310/10001 | Loss: 3.3127 | LR: 5.99e-06 | 1409.8ms/step | 45.4 samples/s
+Step 9320/10001 | Loss: 3.2995 | LR: 5.81e-06 | 1413.6ms/step | 45.3 samples/s
+Step 9330/10001 | Loss: 3.3521 | LR: 5.64e-06 | 1409.4ms/step | 45.4 samples/s
+Step 9340/10001 | Loss: 3.3341 | LR: 5.48e-06 | 1411.0ms/step | 45.4 samples/s
+Step 9350/10001 | Loss: 3.3709 | LR: 5.31e-06 | 1414.9ms/step | 45.2 samples/s
+Step 9360/10001 | Loss: 3.3526 | LR: 5.15e-06 | 1413.4ms/step | 45.3 samples/s
+Step 9370/10001 | Loss: 3.3896 | LR: 4.99e-06 | 1417.6ms/step | 45.1 samples/s
+Step 9380/10001 | Loss: 3.3498 | LR: 4.84e-06 | 1413.4ms/step | 45.3 samples/s
+Step 9390/10001 | Loss: 3.3670 | LR: 4.68e-06 | 1412.1ms/step | 45.3 samples/s
+Step 9400/10001 | Loss: 3.3433 | LR: 4.53e-06 | 1413.5ms/step | 45.3 samples/s
+Step 9410/10001 | Loss: 3.3785 | LR: 4.38e-06 | 1415.2ms/step | 45.2 samples/s
+Step 9420/10001 | Loss: 3.3352 | LR: 4.24e-06 | 1419.4ms/step | 45.1 samples/s
+Step 9430/10001 | Loss: 3.3475 | LR: 4.09e-06 | 1416.9ms/step | 45.2 samples/s
+Step 9440/10001 | Loss: 3.3522 | LR: 3.95e-06 | 1416.4ms/step | 45.2 samples/s
+Step 9450/10001 | Loss: 3.3449 | LR: 3.81e-06 | 1419.4ms/step | 45.1 samples/s
+Step 9460/10001 | Loss: 3.3735 | LR: 3.67e-06 | 1416.7ms/step | 45.2 samples/s
+Step 9470/10001 | Loss: 3.3274 | LR: 3.54e-06 | 1419.4ms/step | 45.1 samples/s
+Step 9480/10001 | Loss: 3.3527 | LR: 3.41e-06 | 1423.3ms/step | 45.0 samples/s
+Step 9490/10001 | Loss: 3.3675 | LR: 3.28e-06 | 1422.7ms/step | 45.0 samples/s
+Step 9500/10001 | Loss: 3.3572 | LR: 3.15e-06 | 1422.2ms/step | 45.0 samples/s
+Validation at step 9500: Loss = 3.3505, Perplexity = 28.52
+Step 9510/10001 | Loss: 3.3633 | LR: 3.03e-06 | 27745.7ms/step | 2.3 samples/s
+Step 9520/10001 | Loss: 3.3442 | LR: 2.91e-06 | 1416.6ms/step | 45.2 samples/s
+Step 9530/10001 | Loss: 3.3353 | LR: 2.79e-06 | 1417.3ms/step | 45.2 samples/s
+Step 9540/10001 | Loss: 3.3499 | LR: 2.67e-06 | 1420.3ms/step | 45.1 samples/s
+Step 9550/10001 | Loss: 3.3837 | LR: 2.56e-06 | 1415.2ms/step | 45.2 samples/s
+Step 9560/10001 | Loss: 3.3397 | LR: 2.44e-06 | 1415.9ms/step | 45.2 samples/s
+Step 9570/10001 | Loss: 3.3443 | LR: 2.33e-06 | 1419.3ms/step | 45.1 samples/s
+Step 9580/10001 | Loss: 3.3392 | LR: 2.23e-06 | 1422.0ms/step | 45.0 samples/s
+Step 9590/10001 | Loss: 3.3672 | LR: 2.12e-06 | 1419.5ms/step | 45.1 samples/s
+Step 9600/10001 | Loss: 3.3518 | LR: 2.02e-06 | 1416.2ms/step | 45.2 samples/s
+Step 9610/10001 | Loss: 3.3286 | LR: 1.92e-06 | 1415.9ms/step | 45.2 samples/s
+Step 9620/10001 | Loss: 3.3443 | LR: 1.82e-06 | 1421.9ms/step | 45.0 samples/s
+Step 9630/10001 | Loss: 3.3665 | LR: 1.73e-06 | 1502.4ms/step | 42.6 samples/s
+Step 9640/10001 | Loss: 3.3635 | LR: 1.64e-06 | 1418.0ms/step | 45.1 samples/s
+Step 9650/10001 | Loss: 3.3557 | LR: 1.55e-06 | 1415.5ms/step | 45.2 samples/s
+Step 9660/10001 | Loss: 3.3377 | LR: 1.46e-06 | 1414.4ms/step | 45.2 samples/s
+Step 9670/10001 | Loss: 3.3231 | LR: 1.38e-06 | 1417.8ms/step | 45.1 samples/s
+Step 9680/10001 | Loss: 3.3291 | LR: 1.30e-06 | 1416.0ms/step | 45.2 samples/s
+Step 9690/10001 | Loss: 3.3578 | LR: 1.22e-06 | 1412.8ms/step | 45.3 samples/s
+Step 9700/10001 | Loss: 3.3529 | LR: 1.14e-06 | 1409.4ms/step | 45.4 samples/s
+Step 9710/10001 | Loss: 3.3550 | LR: 1.06e-06 | 1410.5ms/step | 45.4 samples/s
+Step 9720/10001 | Loss: 3.3581 | LR: 9.93e-07 | 1411.1ms/step | 45.4 samples/s
+Step 9730/10001 | Loss: 3.3479 | LR: 9.24e-07 | 1414.1ms/step | 45.3 samples/s
+Step 9740/10001 | Loss: 3.3374 | LR: 8.57e-07 | 1420.8ms/step | 45.0 samples/s
+Step 9750/10001 | Loss: 3.3641 | LR: 7.92e-07 | 1435.7ms/step | 44.6 samples/s
+Validation at step 9750: Loss = 3.3505, Perplexity = 28.52
+Step 9760/10001 | Loss: 3.3441 | LR: 7.31e-07 | 27778.7ms/step | 2.3 samples/s
+Saved checkpoint to checkpoints/run_20260430_021105/checkpoint-9764

epfl_final/wandb/run-20260430_021108-ngpqwy44/files/requirements.txt

@@ -0,0 +1,97 @@
+pytz==2026.1.post1
+nvidia-cusparselt-cu13==0.8.0
+mpmath==1.3.0
+cuda-toolkit==13.0.2
+zstandard==0.25.0
+xxhash==3.7.0
+urllib3==2.6.3
+tzdata==2026.2
+typing_extensions==4.15.0
+triton==3.6.0
+tqdm==4.67.3
+torchinfo==1.8.0
+sympy==1.14.0
+smmap==5.0.3
+six==1.17.0
+shellingham==1.5.4
+safetensors==0.7.0
+regex==2026.4.4
+PyYAML==6.0.3
+Pygments==2.20.0
+pyarrow==24.0.0
+psutil==7.2.2
+protobuf==7.34.1
+propcache==0.4.1
+platformdirs==4.9.6
+packaging==26.2
+nvidia-nvtx==13.0.85
+nvidia-nvshmem-cu13==3.4.5
+nvidia-nvjitlink==13.0.88
+nvidia-nccl-cu13==2.28.9
+nvidia-curand==10.4.0.35
+nvidia-cufile==1.15.1.6
+nvidia-cuda-runtime==13.0.96
+nvidia-cuda-nvrtc==13.0.88
+nvidia-cuda-cupti==13.0.85
+nvidia-cublas==13.1.0.3
+numpy==2.2.6
+networkx==3.4.2
+mdurl==0.1.2
+MarkupSafe==3.0.3
+idna==3.13
+hf-xet==1.4.3
+h11==0.16.0
+fsspec==2026.2.0
+frozenlist==1.8.0
+filelock==3.29.0
+dill==0.4.1
+cuda-pathfinder==1.5.4
+click==8.3.3
+charset-normalizer==3.4.7
+certifi==2026.4.22
+attrs==26.1.0
+async-timeout==5.0.1
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+typing-inspection==0.4.2
+sentry-sdk==2.58.0
+requests==2.33.1
+python-dateutil==2.9.0.post0
+pydantic_core==2.46.3
+nvidia-cusparse==12.6.3.3
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.19.0.56
+multiprocess==0.70.19
+multidict==6.7.1
+markdown-it-py==4.0.0
+Jinja2==3.1.6
+httpcore==1.0.9
+gitdb==4.0.12
+exceptiongroup==1.3.1
+cuda-bindings==13.2.0
+aiosignal==1.4.0
+yarl==1.23.0
+rich==15.0.0
+pydantic==2.13.3
+pandas==2.3.3
+nvidia-cusolver==12.0.4.66
+GitPython==3.1.49
+anyio==4.13.0
+wandb==0.26.1
+typer==0.25.0
+httpx==0.28.1
+aiohttp==3.13.5
+torch==2.11.0
+huggingface_hub==1.12.2
+tokenizers==0.22.2
+datasets==4.8.5
+accelerate==1.13.0
+transformers==5.7.0
+PyGObject==3.42.1
+dbus-python==1.2.18
+distro==1.7.0
+pip==22.0.2
+setuptools==59.6.0
+ssh-import-id==5.11
+wheel==0.37.1

epfl_final/wandb/run-20260430_021108-ngpqwy44/files/wandb-metadata.json

@@ -0,0 +1,36 @@
+{
+  "os":  "Linux-6.8.0-59-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.12",
+  "startedAt":  "2026-04-30T02:11:08.532189Z",
+  "program":  "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
+  "codePath":  "SparseAtt/train.py",
+  "codePathLocal":  "SparseAtt/train.py",
+  "email":  "ruben.navasardyan@mail.utoronto.ca",
+  "root":  "/mloscratch/homes/navasard/sparse",
+  "host":  "h100-workspace-5-0-3",
+  "executable":  "/usr/bin/python3",
+  "cpu_count":  96,
+  "cpu_count_logical":  192,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  1,
+  "disk":  {
+    "/":  {
+      "total":  "9371709865984",
+      "used":  "4381359190016"
+    }
+  },
+  "memory":  {
+    "total":  "1623005847552"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-9a903f43-9348-6fa6-bcf7-e12bd01803a7"
+    }
+  ],
+  "cudaVersion":  "13.0",
+  "writerId":  "7fambcx7u8b13zjiikjiufanttbl1jhf"
+}

epfl_final/wandb/run-20260430_021108-ngpqwy44/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/loss":3.3504738580584528,"speed/steps_per_sec":0.03599874110695133,"speed/ms_per_step":27778.749179840088,"eval/perplexity":28.51624309432053,"_wandb":{"runtime":24613},"_timestamp":1.777539671983468e+09,"train/epoch":0,"train/global_step":9760,"speed/samples_per_sec":2.303919430844885,"train/loss":3.3440643697977066,"_step":9760,"_runtime":24613.62177148,"train/learning_rate":7.305906022035125e-07}

epfl_final/wandb/run-20260430_021108-ngpqwy44/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-30 02:11:08,550 INFO    MainThread:283 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-04-30 02:11:08,550 INFO    MainThread:283 [wandb_setup.py:_flush():81] Configure stats pid to 283
+2026-04-30 02:11:08,550 INFO    MainThread:283 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 02:11:08,550 INFO    MainThread:283 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_021108-ngpqwy44/logs/debug.log
+2026-04-30 02:11:08,551 INFO    MainThread:283 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_021108-ngpqwy44/logs/debug-internal.log
+2026-04-30 02:11:08,551 INFO    MainThread:283 [wandb_init.py:init():850] calling init triggers
+2026-04-30 02:11:08,551 INFO    MainThread:283 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 2048, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_021105', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10001, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'flash_attention': True, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 7, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, '_wandb': {}}
+2026-04-30 02:11:08,551 INFO    MainThread:283 [wandb_init.py:init():898] starting backend
+2026-04-30 02:11:08,780 INFO    MainThread:283 [wandb_init.py:init():913] sending inform_init request
+2026-04-30 02:11:09,103 INFO    MainThread:283 [wandb_init.py:init():918] backend started and connected
+2026-04-30 02:11:09,107 INFO    MainThread:283 [wandb_init.py:init():988] updated telemetry
+2026-04-30 02:11:09,108 INFO    MainThread:283 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-04-30 02:11:09,580 INFO    MainThread:283 [wandb_init.py:init():1056] starting run threads in backend
+2026-04-30 02:11:09,666 INFO    MainThread:283 [wandb_run.py:_console_start():2554] atexit reg
+2026-04-30 02:11:09,666 INFO    MainThread:283 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-04-30 02:11:09,666 INFO    MainThread:283 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-04-30 02:11:09,666 INFO    MainThread:283 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-04-30 02:11:09,669 INFO    MainThread:283 [wandb_init.py:init():1094] run started, returning control to user process
+2026-04-30 09:01:23,202 INFO    MainThread:283 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/ngpqwy44
+2026-04-30 09:01:23,202 INFO    MainThread:283 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-04-30 09:01:23,203 INFO    MainThread:283 [wandb_run.py:_restore():2502] restore
+2026-04-30 09:01:23,203 INFO    MainThread:283 [wandb_run.py:_restore():2508] restore done
+2026-04-30 09:01:24,303 INFO    MainThread:283 [wandb_run.py:_footer_sync_info():3889] logging synced files

epfl_final/wandb/run-20260430_021108-ngpqwy44/run-ngpqwy44.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:616d29edf6215680d6e43a556a1ce769de1eeea6dd9c46e9b33a9df15f5be16c
+size 4154731

epfl_final/wandb/run-20260430_021315-cqkg8w61/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_setup.py:_flush():81] Configure stats pid to 217
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_021315-cqkg8w61/logs/debug.log
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/wandb/run-20260430_021315-cqkg8w61/logs/debug-internal.log
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_init.py:init():850] calling init triggers
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
+config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/slimpajama_packed_train/tokenized', 'val_path': '/mloscratch/homes/navasard/data/slimpajama_packed_val/tokenized', 'max_seq_length': 2048, 'num_workers': 8, 'output_dir': './checkpoints/run_20260430_021312', 'resume_from_checkpoint': None, 'num_epochs': 1, 'max_steps': 10001, 'per_device_batch_size': 8, 'gradient_accumulation_steps': 8, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 100, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 2500, 'eval_steps': 250, 'eval_batches': 500, 'logging_steps': 10, 'seed': 42, 'verbose': True, 'checkpoint_activations': False, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': None, 'wandb_tags': ['qwen3', '0.6b'], 'flash_attention': True, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 3, 'sparse_mask_pool_size': 1000, 'sparse_val_dense_compare': False, '_wandb': {}}
+2026-04-30 02:13:15,980 INFO    MainThread:217 [wandb_init.py:init():898] starting backend
+2026-04-30 02:13:16,198 INFO    MainThread:217 [wandb_init.py:init():913] sending inform_init request
+2026-04-30 02:13:16,494 INFO    MainThread:217 [wandb_init.py:init():918] backend started and connected
+2026-04-30 02:13:16,497 INFO    MainThread:217 [wandb_init.py:init():988] updated telemetry
+2026-04-30 02:13:16,498 INFO    MainThread:217 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
+2026-04-30 02:13:17,205 INFO    MainThread:217 [wandb_init.py:init():1056] starting run threads in backend
+2026-04-30 02:13:17,289 INFO    MainThread:217 [wandb_run.py:_console_start():2554] atexit reg
+2026-04-30 02:13:17,289 INFO    MainThread:217 [wandb_run.py:_redirect():2403] redirect: wrap_raw
+2026-04-30 02:13:17,289 INFO    MainThread:217 [wandb_run.py:_redirect():2472] Wrapping output streams.
+2026-04-30 02:13:17,289 INFO    MainThread:217 [wandb_run.py:_redirect():2495] Redirects installed.
+2026-04-30 02:13:17,293 INFO    MainThread:217 [wandb_init.py:init():1094] run started, returning control to user process
+2026-04-30 08:40:43,150 INFO    MainThread:217 [wandb_run.py:_finish():2321] finishing run ruben-navasardyan4869-epfl/stochastic_sparsity/cqkg8w61
+2026-04-30 08:40:43,151 INFO    MainThread:217 [wandb_run.py:_atexit_cleanup():2520] got exitcode: 0
+2026-04-30 08:40:43,151 INFO    MainThread:217 [wandb_run.py:_restore():2502] restore
+2026-04-30 08:40:43,152 INFO    MainThread:217 [wandb_run.py:_restore():2508] restore done
+2026-04-30 08:40:45,508 INFO    MainThread:217 [wandb_run.py:_footer_sync_info():3889] logging synced files