Archive current RCP 365M learnable-gain recipe and log snapshot

epfl_final/SparseAtt/configs/rcp_20260927/365m_dense_qkgainfp32_s42_60b_1gpu.json

@@ -0,0 +1,64 @@
+{
+  "model": {
+    "path": "../../qwen3_06b_base",
+    "dtype": "bfloat16",
+    "init_from_scratch": true,
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 6,
+    "num_key_value_heads": 2,
+    "rope_theta": 1000000.0,
+    "qk_norm": "on",
+    "qk_gain_fp32": true,
+    "positional": "rope"
+  },
+  "data": {
+    "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+    "val_path": null,
+    "seed": 42,
+    "max_seq_length": 8192,
+    "num_workers": 4
+  },
+  "training": {
+    "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/rcp_365m_dense_qkgainfp32_60b_seed42_20260927",
+    "resume_from_checkpoint": null,
+    "force_new_run": false,
+    "num_epochs": 3,
+    "max_steps": 366211,
+    "per_device_batch_size": 1,
+    "gradient_accumulation_steps": 20,
+    "learning_rate": 0.0005,
+    "weight_decay": 0.01,
+    "warmup_steps": 750,
+    "lr_scheduler": "cosine",
+    "max_grad_norm": 1.0,
+    "save_steps": 5000,
+    "eval_steps": 1000,
+    "eval_batches": 10,
+    "logging_steps": 100,
+    "seed": 42,
+    "verbose": false,
+    "checkpoint_activations": false
+  },
+  "distributed": {
+    "strategy": "ddp",
+    "mixed_precision": "bf16"
+  },
+  "wandb": {
+    "enabled": false,
+    "project": "stochastic_sparsity",
+    "name": "RCP 365m dense qkgainfp32 seed42 60B single A100 20260927",
+    "tags": [
+      "final_20260924",
+      "part_a",
+      "dense",
+      "qkgainfp32",
+      "seed42"
+    ]
+  },
+  "attention": {
+    "type": "dense",
+    "dense_mode": false
+  }
+}

epfl_final/train_logs/rcp_365m_dense_qkgainfp32_s42_60b_20260927.log

@@ -0,0 +1,306 @@
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/rcp_365m_dense_qkgainfp32_60b_seed42_20260927/run_20260927_230247
+Running config:
+{
+  "model_path": "../../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 4,
+  "data_seed": 42,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/rcp_365m_dense_qkgainfp32_60b_seed42_20260927/run_20260927_230247",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 3,
+  "max_steps": 366211,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 5000,
+  "eval_steps": 1000,
+  "eval_batches": 10,
+  "logging_steps": 100,
+  "seed": 42,
+  "verbose": false,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "RCP 365m dense qkgainfp32 seed42 60B single A100 20260927",
+  "wandb_tags": [
+    "final_20260924",
+    "part_a",
+    "dense",
+    "qkgainfp32",
+    "seed42"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_theta": 1000000.0,
+  "rope_train_position_scale_set": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "attention_type": "dense",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 1,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from ../../qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000.0
+  rope_train_position_scale_set: None
+  attention_type: dense
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+================================================================================
+
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 173, in getuser
+    return pwd.getpwuid(os.getuid())[0]
+           ~~~~~~~~~~~~^^^^^^^^^^^^^
+KeyError: 'getpwuid(): uid not found: 285818'
+
+The above exception was the direct cause of the following exception:
+
+Traceback (most recent call last):
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1593, in <module>
+    main()
+    ~~~~^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1590, in main
+    train(config)
+    ~~~~~^^^^^^^^
+  File "/mloscratch/homes/navasard/sparse/SparseAtt/train.py", line 1158, in train
+    model = torch.compile(model)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/__init__.py", line 2758, in compile
+    return torch._dynamo.optimize(
+           ~~~~~~~~~~~~~~~~~~~~~~^
+        backend=backend,
+        ^^^^^^^^^^^^^^^^
+    ...<3 lines>...
+        guard_filter_fn=guard_filter_fn,
+        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    )(model)  # type: ignore[return-value]
+    ^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1488, in optimize
+    return _optimize(rebuild_ctx, *args, **kwargs)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1564, in _optimize
+    backend = get_compiler_fn(backend)
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/eval_frame.py", line 1325, in get_compiler_fn
+    from .repro.after_dynamo import wrap_backend_debug
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/repro/after_dynamo.py", line 33, in <module>
+    from torch._dynamo.debug_utils import (
+    ...<15 lines>...
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/debug_utils.py", line 43, in <module>
+    from torch._dynamo.testing import rand_strided
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/testing.py", line 33, in <module>
+    from torch._dynamo.backends.debugging import aot_eager
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_dynamo/backends/debugging.py", line 34, in <module>
+    from functorch.compile import min_cut_rematerialization_partition
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/functorch/compile/__init__.py", line 2, in <module>
+    from torch._functorch.aot_autograd import (
+    ...<10 lines>...
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/aot_autograd.py", line 38, in <module>
+    from ._aot_autograd.autograd_cache import (  # noqa: F401
+    ...<4 lines>...
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_functorch/_aot_autograd/autograd_cache.py", line 27, in <module>
+    from torch._inductor.codecache import (
+    ...<11 lines>...
+    )
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/codecache.py", line 2753, in <module>
+    _HEADER_DIR = os.path.join(default_cache_dir(), "precompiled_headers")
+                               ~~~~~~~~~~~~~~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/site-packages/torch/_inductor/runtime/cache_dir_utils.py", line 23, in default_cache_dir
+    sanitized_username = re.sub(r'[\\/:*?"<>|]', "_", getpass.getuser())
+                                                      ~~~~~~~~~~~~~~~^^
+  File "/mloscratch/homes/navasard/conda/lib/python3.13/getpass.py", line 175, in getuser
+    raise OSError('No username set in the environment') from e
+OSError: No username set in the environment
+New run: Checkpoints will be saved to /mloscratch/homes/navasard/sparse/checkpoints/rcp_365m_dense_qkgainfp32_60b_seed42_20260927/run_20260927_230351
+Running config:
+{
+  "model_path": "../../qwen3_06b_base",
+  "model_dtype": "bfloat16",
+  "init_from_scratch": true,
+  "train_path": "/mloscratch/homes/navasard/data/fineweb_large/tokenized",
+  "val_path": null,
+  "max_seq_length": 8192,
+  "num_workers": 4,
+  "data_seed": 42,
+  "output_dir": "/mloscratch/homes/navasard/sparse/checkpoints/rcp_365m_dense_qkgainfp32_60b_seed42_20260927/run_20260927_230351",
+  "resume_from_checkpoint": null,
+  "force_new_run": false,
+  "num_epochs": 3,
+  "max_steps": 366211,
+  "per_device_batch_size": 1,
+  "gradient_accumulation_steps": 20,
+  "learning_rate": 0.0005,
+  "weight_decay": 0.01,
+  "warmup_steps": 750,
+  "lr_scheduler": "cosine",
+  "max_grad_norm": 1.0,
+  "save_steps": 5000,
+  "eval_steps": 1000,
+  "eval_batches": 10,
+  "logging_steps": 100,
+  "seed": 42,
+  "verbose": false,
+  "checkpoint_activations": false,
+  "resume_micro_step_override": null,
+  "strategy": "ddp",
+  "mixed_precision": "bf16",
+  "wandb_enabled": false,
+  "wandb_project": "stochastic_sparsity",
+  "wandb_name": "RCP 365m dense qkgainfp32 seed42 60B single A100 20260927",
+  "wandb_tags": [
+    "final_20260924",
+    "part_a",
+    "dense",
+    "qkgainfp32",
+    "seed42"
+  ],
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "num_attention_heads": 6,
+  "num_key_value_heads": 2,
+  "num_hidden_layers": 28,
+  "head_dim": null,
+  "attention_dropout": null,
+  "rope_theta": 1000000.0,
+  "rope_train_position_scale_set": null,
+  "positional": "rope",
+  "qk_norm": "on",
+  "qk_gain_fp32": true,
+  "attention_type": "dense",
+  "sparse_block_size": 128,
+  "sparse_num_random_blocks": 1,
+  "sparse_mask_pool_size": 1000,
+  "sparse_val_dense_compare": false,
+  "sparse_layer_indices": null,
+  "sparse_mask_cache_dir": null,
+  "sparse_mask_seed": 0,
+  "sparse_mixed_nrb_set": null,
+  "dense_mode": false,
+  "dense_step_intervals": null
+}
+Loading tokenizer from ../../qwen3_06b_base...
+Initializing model from scratch (random weights)...
+Checkpoint activations: False
+
+
+================================================================================
+MODEL ARCHITECTURE
+================================================================================
+Model config:
+  vocab_size: 151936
+  hidden_size: 768
+  num_hidden_layers: 28
+  num_attention_heads: 6
+  num_key_value_heads: 2
+  head_dim: 128
+  intermediate_size: 3072
+  max_position_embeddings: 32768
+  rope_theta: 1000000.0
+  rope_train_position_scale_set: None
+  attention_type: dense
+
+Parameter count:
+  Total parameters: 358,958,848
+  Trainable parameters: 358,958,848
+  Non-trainable parameters: 0
+  Model size (BF16): 0.72 GB
+================================================================================
+
+Loading dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Using pre-tokenized dataset (no runtime tokenization)
+Loading pre-tokenized HF dataset from /mloscratch/homes/navasard/data/fineweb_large/tokenized...
+Loaded 12748481 pre-tokenized examples (stored_len=2048, pack_factor=4, packed_examples=3187120)
+Dataset split: 3027764 train, 159356 validation
+DistributedSampler seed: 42 (run seed)
+Training examples (packed, train split):  3027764
+Validation examples (packed, val split):  159356
+Per-device batch size:                    1
+World size (GPUs):                        1
+Gradient accumulation steps:              20
+Effective batch size (seqs/step):         20
+Sequence length:                          8192
+Tokens per optimizer step:                163,840
+Micro-batches per epoch (per rank):       3027764
+Optimizer steps per epoch:                151388
+Total optimizer steps:                    366211  (capped by max_steps; 2.42 epochs)
+Starting training...
+Switching attention mode for step 1: sparse_stochastic (configured sparse attention)
+Step 100/366211 | Loss: 10.6766 | LR: 6.67e-05 | 4640.3ms/step | 4.3 samples/s
+Step 200/366211 | Loss: 7.4739 | LR: 1.33e-04 | 3930.7ms/step | 5.1 samples/s
+Step 300/366211 | Loss: 6.4732 | LR: 2.00e-04 | 3941.6ms/step | 5.1 samples/s
+Step 400/366211 | Loss: 5.9900 | LR: 2.67e-04 | 3939.9ms/step | 5.1 samples/s
+Step 500/366211 | Loss: 5.6358 | LR: 3.33e-04 | 3940.0ms/step | 5.1 samples/s
+Step 600/366211 | Loss: 5.3014 | LR: 4.00e-04 | 4019.3ms/step | 5.0 samples/s
+Step 700/366211 | Loss: 4.9687 | LR: 4.67e-04 | 3986.2ms/step | 5.0 samples/s
+Step 800/366211 | Loss: 4.6597 | LR: 5.00e-04 | 3980.2ms/step | 5.0 samples/s
+Step 900/366211 | Loss: 4.4402 | LR: 5.00e-04 | 3938.8ms/step | 5.1 samples/s
+Step 1000/366211 | Loss: 4.2723 | LR: 5.00e-04 | 3939.1ms/step | 5.1 samples/s
+Validation at step 1000: Loss = 4.2365, Perplexity = 69.17
+Step 1100/366211 | Loss: 4.1459 | LR: 5.00e-04 | 4321.4ms/step | 4.6 samples/s
+Step 1200/366211 | Loss: 4.0458 | LR: 5.00e-04 | 3947.4ms/step | 5.1 samples/s
+Step 1300/366211 | Loss: 3.9735 | LR: 5.00e-04 | 3946.7ms/step | 5.1 samples/s
+Step 1400/366211 | Loss: 3.8998 | LR: 5.00e-04 | 3946.1ms/step | 5.1 samples/s
+Step 1500/366211 | Loss: 3.8672 | LR: 5.00e-04 | 3945.7ms/step | 5.1 samples/s
+Step 1600/366211 | Loss: 3.8066 | LR: 5.00e-04 | 3944.2ms/step | 5.1 samples/s
+Step 1700/366211 | Loss: 3.7503 | LR: 5.00e-04 | 3943.7ms/step | 5.1 samples/s
+Step 1800/366211 | Loss: 3.7114 | LR: 5.00e-04 | 3945.0ms/step | 5.1 samples/s
+Step 1900/366211 | Loss: 3.6941 | LR: 5.00e-04 | 3946.2ms/step | 5.1 samples/s
+Step 2000/366211 | Loss: 3.6601 | LR: 5.00e-04 | 3944.8ms/step | 5.1 samples/s
+Validation at step 2000: Loss = 3.6574, Perplexity = 38.76
+Step 2100/366211 | Loss: 3.6219 | LR: 5.00e-04 | 4075.0ms/step | 4.9 samples/s
+Step 2200/366211 | Loss: 3.6010 | LR: 5.00e-04 | 3947.5ms/step | 5.1 samples/s
+Step 2300/366211 | Loss: 3.5732 | LR: 5.00e-04 | 3948.0ms/step | 5.1 samples/s
+Step 2400/366211 | Loss: 3.5509 | LR: 5.00e-04 | 3947.7ms/step | 5.1 samples/s
+Step 2500/366211 | Loss: 3.5410 | LR: 5.00e-04 | 3949.1ms/step | 5.1 samples/s
+Step 2600/366211 | Loss: 3.5235 | LR: 5.00e-04 | 3951.8ms/step | 5.1 samples/s
+Step 2700/366211 | Loss: 3.5097 | LR: 5.00e-04 | 3952.1ms/step | 5.1 samples/s
+Step 2800/366211 | Loss: 3.4834 | LR: 5.00e-04 | 3945.9ms/step | 5.1 samples/s
+Step 2900/366211 | Loss: 3.4633 | LR: 5.00e-04 | 3946.3ms/step | 5.1 samples/s
+Step 3000/366211 | Loss: 3.4536 | LR: 5.00e-04 | 3944.8ms/step | 5.1 samples/s
+Validation at step 3000: Loss = 3.4646, Perplexity = 31.96