Add vanilla 476M 7B final model weights and configurations (step 26703)

20260916-204226_vanilla_476m_7b/config.json

@@ -0,0 +1,154 @@
+{
+  "execution_architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "objective": "one_pass_causal_lm",
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "width_scale": 22,
+  "hidden_size": 1408,
+  "num_layers": 20,
+  "head_dim": 128,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "ffn_hidden_size": 5632,
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "norm_epsilon": 1e-06,
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "rope_base": 10000,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "bias": false,
+  "dropout": 0.0,
+  "residual_multiplier": 1.0,
+  "residual_and_norm_scale_dtype": "float32",
+  "projection_dtype": "bfloat16 (production)",
+  "tied_embeddings": false,
+  "vocab_size": 32001,
+  "padded_vocab_size": 32064,
+  "logit_softcap": 15.0,
+  "crop_padding_before_loss": true,
+  "train_passes": 1,
+  "eval_passes": 1,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "parameter_counts": {
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 566144128,
+    "feedback_parameters": 0
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "feedback_enabled": false,
+  "feedback_pass_mixture": {
+    "1": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 26703,
+  "checkpoint_version": 3.0,
+  "training_run": "20260916-204226_vanilla_476m_7b",
+  "training_code_revision": "a57216c3768aebd1226a30fa573b09530a9537f2",
+  "training_tokens": 7000031232,
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "sequence_length": 2048,
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "loading_note": "Native loopTF FullBandwidthMegatronModel state_dict names, not a Transformers AutoModel export. Load safetensors tensors, restore the null extra-state entries listed here, then load_state_dict into the matching model/profile. Optimizer, scheduler and RNG state are excluded.",
+  "tokenizer": {
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "additional_token": "<|assistant_end|>",
+    "active_vocab_size": 32001
+  }
+}

20260916-204226_vanilla_476m_7b/launch_config.env

@@ -0,0 +1,90 @@
+# Vanilla calibration through Full-Bandwidth infrastructure; see docs/PAPER2604_VANILLA.md.
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online
+RESUME_POLICY=fresh
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# One node by default; the existing multi-node launcher supplies NNODES/NODE_RANK.
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+FEEDBACK_PASS_MIXTURE=1:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=1
+FEEDBACK_LOSS_WEIGHT=0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=2000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

20260916-204226_vanilla_476m_7b/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da0cded7c2f112deaff51b85bfab738d9f45c68c4bf35c64848fabab364b8bc7
+size 1132423808

20260916-204226_vanilla_476m_7b/training_recipe.json

@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "execution_architecture": "full_bandwidth",
+    "backbone_profile": "paper2604_vanilla",
+    "objective": "one_pass_causal_lm",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "train_passes": 1,
+    "eval_passes": 1,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 566144128,
+      "feedback_parameters": 0
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "feedback_enabled": false,
+    "feedback_pass_mixture": {
+      "1": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}