Hugging Face's logo

Siyuc
/
loopTF-workspace

Safetensors
Model card Files
xet
 Community
Siyuc commited on
Commit
de1e5bd
·
verified ·
1 Parent(s): 062d2eb

Add unified FBW checkpoint paths and naming map

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/NAMING.json +10 -0
  2. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/README.md +35 -0
  3. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/config.json +289 -0
  4. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/launch_config.env +107 -0
  5. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/manifest.json +30 -0
  6. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/model.safetensors +3 -0
  7. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/training_recipe.json +142 -0
  8. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/README.md +35 -0
  9. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/config.json +289 -0
  10. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/launch_config.env +107 -0
  11. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/manifest.json +30 -0
  12. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/model.safetensors +3 -0
  13. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/training_recipe.json +142 -0
  14. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/README.md +35 -0
  15. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/config.json +289 -0
  16. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/launch_config.env +107 -0
  17. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/manifest.json +30 -0
  18. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/model.safetensors +3 -0
  19. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/training_recipe.json +142 -0
  20. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/README.md +35 -0
  21. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/config.json +289 -0
  22. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/launch_config.env +107 -0
  23. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/manifest.json +30 -0
  24. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/model.safetensors +3 -0
  25. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/training_recipe.json +142 -0
  26. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/README.md +35 -0
  27. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/config.json +289 -0
  28. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/launch_config.env +107 -0
  29. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/manifest.json +30 -0
  30. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/model.safetensors +3 -0
  31. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/training_recipe.json +142 -0
  32. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/README.md +35 -0
  33. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/config.json +289 -0
  34. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/launch_config.env +107 -0
  35. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/manifest.json +30 -0
  36. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/model.safetensors +3 -0
  37. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/training_recipe.json +142 -0
  38. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/README.md +35 -0
  39. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/config.json +289 -0
  40. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/launch_config.env +107 -0
  41. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/manifest.json +30 -0
  42. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/model.safetensors +3 -0
  43. FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/training_recipe.json +142 -0
  44. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/NAMING.json +10 -0
  45. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/README.md +50 -0
  46. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/config.json +1339 -0
  47. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/launch_config.env +6 -0
  48. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/manifest.json +25 -0
  49. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/model.safetensors +3 -0
  50. FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/training_recipe.json +133 -0
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/NAMING.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "wandb_id": "ljsfjz37",
3
+ "previous_wandb_name": "fbw-layers2-11-20-k3-s42-7b--ljsfjz37",
4
+ "training_run_name": "fbw_L2L11L20_7b_seed42_20260920-023045",
5
+ "canonical_name": "FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37",
6
+ "architecture": "full_bandwidth",
7
+ "hf_old_paths": [
8
+ "fbw_L2L11L20_7b_seed42_20260920-023045"
9
+ ]
10
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 12000
2
+
3
+ Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_12000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 12000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 1536000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 12000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 3145728000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 12000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "ecfcad9dcd20100fe9c87281508367495a2e47f6ec745b20fad8455b5879fd1f",
6
+ "size": 2699
7
+ },
8
+ "config.json": {
9
+ "sha256": "6e6a9ced87ea8f55c9ab0533b901e84e97991d5aa0838f0b29ee213e9639793a",
10
+ "size": 12273
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_12000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 16000
2
+
3
+ Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_16000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 16000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 2048000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 16000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 4194304000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 16000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "d3f4a9de16bf052561064000ad77015e88319f11afa1b6c0d01d3628ec9dc4b6",
6
+ "size": 2699
7
+ },
8
+ "config.json": {
9
+ "sha256": "508f1517705df16a8bd79f1b0c9ed5c610679485964d5c670e95fbafa6f83d9a",
10
+ "size": 12273
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_16000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 20000
2
+
3
+ Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_20000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 20000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 2560000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 20000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 5242880000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 20000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "e5c48096d83b5fbb9c8ba691b37f3f483a5cf1c67ef37d9dd0bc90c072fe1b18",
6
+ "size": 2699
7
+ },
8
+ "config.json": {
9
+ "sha256": "0b26f395c2623ab007efcafec34a04fd0b3238d47d690b1bc2cf2cb9d2116737",
10
+ "size": 12273
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_20000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 24000
2
+
3
+ Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_24000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 24000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 3072000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 24000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 6291456000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 24000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "a2d3faf19e2eea6312eb85940944b0afbe84dfcc25d302616f2c66549bfe8ad2",
6
+ "size": 2699
7
+ },
8
+ "config.json": {
9
+ "sha256": "5619a51676440252ea6ff2c78de18c37322142cacc04d0188457faa0e28c4d8d",
10
+ "size": 12273
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_24000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 26703
2
+
3
+ Final checkpoint at step **26,703**, after **7,000,031,232 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_26703")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 26703,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 3417984,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": true,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 26703,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 7000031232,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 26703,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "d5a7044737b80db05eaf39029237d2df47cfd6135677146decb1997732ae5266",
6
+ "size": 2692
7
+ },
8
+ "config.json": {
9
+ "sha256": "0be3be4a6e3da7092a2bc29821ced20389b0298ede9879790c9762c65ff47ea3",
10
+ "size": 12272
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_26703",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 4000
2
+
3
+ Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_4000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 4000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 512000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 4000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 1048576000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 4000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "754c86bf86065e0f121d51b1f3046858ead7aaea160a46caa6db1ddc72cb127d",
6
+ "size": 2696
7
+ },
8
+ "config.json": {
9
+ "sha256": "880053416c618d5a37cb0de24ba8dfdd88418832d970f89d2e55311474136a81",
10
+ "size": 12270
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_4000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/README.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fbw_L2L11L20_7b_seed42_20260920-023045 — step 8000
2
+
3
+ Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
4
+
5
+ Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
6
+
7
+ Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
8
+
9
+ The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
10
+
11
+ ## Loading
12
+
13
+ ```python
14
+ import json
15
+ from pathlib import Path
16
+ import torch
17
+ from safetensors.torch import load_file
18
+ export_dir = Path("/path/to/step_8000")
19
+ config = json.loads((export_dir / "config.json").read_text())
20
+ state = load_file(str(export_dir / "model.safetensors"), device="cpu")
21
+ from types import SimpleNamespace
22
+ from loop_transformer_backends.benchmarks.models import build_native
23
+ model, metadata = build_native({
24
+ "args": SimpleNamespace(**config["native_loading_args"]),
25
+ "model": state | config["non_tensor_state_dict_entries"],
26
+ "iteration": config["checkpoint_iteration"],
27
+ }) # Existing project inference mapping, strict=True, assign=True internally.
28
+ model.eval()
29
+ ```
30
+
31
+ Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
32
+
33
+ No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
34
+
35
+ W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/config.json ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation": "squared_relu",
3
+ "attention": "full_causal",
4
+ "attention_output_gate": false,
5
+ "backbone_profile": "paper2604_vanilla",
6
+ "bias": false,
7
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
8
+ "checkpoint_iteration": 8000,
9
+ "checkpoint_version": 3.0,
10
+ "consumed_train_samples": 1024000,
11
+ "crop_padding_before_loss": true,
12
+ "dropout": 0.0,
13
+ "effective_overrides": {
14
+ "EXIT_INTERVAL": 26703,
15
+ "FEEDBACK_EVAL_PASSES": 3,
16
+ "FEEDBACK_JITTER_SIGMA": 0.0,
17
+ "FEEDBACK_LAYERS": "2,11,20",
18
+ "FEEDBACK_LOG_HIDDEN_CHANGE": false,
19
+ "FEEDBACK_LOSS_WEIGHT": 1.0,
20
+ "FEEDBACK_NORMUON_BETA2": 0.95,
21
+ "FEEDBACK_PASS_MIXTURE": "3:1.0",
22
+ "FEEDBACK_PREFIX_MIXIN": false,
23
+ "FEEDBACK_PROGRESSIVE_STAGES": "[]",
24
+ "FEEDBACK_SCALAR_LR": 0.0005,
25
+ "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
26
+ "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
27
+ "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
28
+ "GLOBAL_BATCH_SIZE": 128,
29
+ "MICRO_BATCH_SIZE": 8,
30
+ "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
31
+ "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
32
+ "ONLINE_DATASET_SUBSET": "sample-100BT",
33
+ "ONLINE_SOURCE_SPLIT": "train",
34
+ "ONLINE_SPLIT_SEED": "260421106",
35
+ "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
36
+ "ONLINE_VALIDATION_FRACTION": "0.001",
37
+ "PP": "1",
38
+ "PRECISION": "bf16",
39
+ "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
40
+ "SAVE_INTERVAL": 4000,
41
+ "SEED": 42,
42
+ "SEQ_LENGTH": "2048",
43
+ "TP": "1",
44
+ "TRAIN_ITERS": 26703,
45
+ "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
46
+ "WANDB_RUN_ID": "ljsfjz37",
47
+ "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
48
+ },
49
+ "eval_passes": 3,
50
+ "execution_architecture": "full_bandwidth",
51
+ "feedback_enabled": true,
52
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
53
+ "feedback_layer_projection_init": "final_layer_identity",
54
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
55
+ "feedback_layers": [
56
+ 2,
57
+ 11,
58
+ 20
59
+ ],
60
+ "feedback_pass_mixture": {
61
+ "3": 1.0
62
+ },
63
+ "feedback_progressive_stages": [],
64
+ "ffn_hidden_size": 5632,
65
+ "final_validation": null,
66
+ "head_dim": 128,
67
+ "hidden_size": 1408,
68
+ "initialization": {
69
+ "embedding_normal_std": 1.0,
70
+ "head_normal_std": 0.001,
71
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
72
+ "norm_scale": 1.0
73
+ },
74
+ "is_final_checkpoint": false,
75
+ "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
76
+ "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
77
+ "logit_softcap": 15.0,
78
+ "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
79
+ "model_level_norms": [
80
+ "embedding",
81
+ "recur",
82
+ "final"
83
+ ],
84
+ "native_inference_adapter": {
85
+ "active_vocab_size": 32001,
86
+ "architecture": "full_bandwidth",
87
+ "context_extension": "native RoPE, no scaling; at most 2x training length",
88
+ "max_supported_length": 4096,
89
+ "model_config": {
90
+ "attention_dropout": 0.0,
91
+ "attn_implementation": "sdpa",
92
+ "backbone_profile": "paper2604_vanilla",
93
+ "gradient_checkpointing": false,
94
+ "hf_config_overrides": {},
95
+ "hidden_size": 1408,
96
+ "initializer_range": 0.02,
97
+ "intermediate_size": 5632,
98
+ "max_position_embeddings": 2048,
99
+ "model_type": "full_bandwidth",
100
+ "num_attention_heads": 11,
101
+ "num_hidden_layers": 20,
102
+ "num_key_value_heads": 11,
103
+ "padded_vocab_size": 32064,
104
+ "paper_full_attention_interval": 6,
105
+ "paper_sliding_window": 0,
106
+ "pretrained_model_name_or_path": null,
107
+ "revision": null,
108
+ "rms_norm_eps": 1e-06,
109
+ "rope_theta": 10000,
110
+ "tie_word_embeddings": false,
111
+ "trust_remote_code": false,
112
+ "vocab_size": 32001
113
+ },
114
+ "policy": {
115
+ "depth_loops": 1,
116
+ "depth_share_kv": false,
117
+ "feedback_layers": [
118
+ 2,
119
+ 11,
120
+ 20
121
+ ],
122
+ "final_adapter": {
123
+ "attention_backend": "sdpa_auto",
124
+ "checkpoint_refiner": true,
125
+ "ffn_size": null,
126
+ "kind": "none"
127
+ },
128
+ "num_passes": 3
129
+ },
130
+ "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
131
+ "step": 8000,
132
+ "training_context_length": 2048
133
+ },
134
+ "native_loading_args": {
135
+ "architecture": "full_bandwidth",
136
+ "attention_dropout": 0.0,
137
+ "aux_mtp_horizon": 1,
138
+ "backbone_profile": "paper2604_vanilla",
139
+ "context_parallel_size": 1,
140
+ "feedback_eval_passes": 3,
141
+ "feedback_layers": "2,11,20",
142
+ "ffn_hidden_size": 5632,
143
+ "hidden_size": 1408,
144
+ "layernorm_epsilon": 1e-06,
145
+ "num_attention_heads": 11,
146
+ "num_layers": 20,
147
+ "num_query_groups": 11,
148
+ "padded_vocab_size": 32064,
149
+ "pipeline_model_parallel_size": 1,
150
+ "rotary_base": 10000,
151
+ "seq_length": 2048,
152
+ "tensor_model_parallel_size": 1,
153
+ "untie_embeddings_and_output_weights": true,
154
+ "vocab_size": 32001
155
+ },
156
+ "non_tensor_state_dict_entries": {
157
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
158
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
159
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
160
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
161
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
162
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
163
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
164
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
165
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
166
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
167
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
168
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
169
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
170
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
171
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
172
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
173
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
174
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
175
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
176
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
177
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
178
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
179
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
180
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
181
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
182
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
183
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
184
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
185
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
186
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
187
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
188
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
189
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
190
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
191
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
192
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
193
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
194
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
195
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
196
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
197
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
198
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
199
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
200
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
201
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
202
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
203
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
204
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
205
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
206
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
207
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
208
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
209
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
210
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
211
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
212
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
213
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
214
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
215
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
216
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
217
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
218
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
219
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
220
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
221
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
222
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
223
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
224
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
225
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
226
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
227
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
228
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
229
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
230
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
231
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
232
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
233
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
234
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
235
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
236
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
237
+ "feedback.state_proj._extra_state": null,
238
+ "feedback.token_gate_proj._extra_state": null,
239
+ "feedback_layer_proj._extra_state": null,
240
+ "output_layer._extra_state": null
241
+ },
242
+ "norm_epsilon": 1e-06,
243
+ "num_attention_heads": 11,
244
+ "num_kv_heads": 11,
245
+ "num_layers": 20,
246
+ "objective": "latent_feedback_causal_lm",
247
+ "padded_vocab_size": 32064,
248
+ "parameter_counts": {
249
+ "feedback_layer_projection_parameters": 5947392,
250
+ "feedback_parameters": 3967744,
251
+ "non_embedding_matrix_formula": 475791360,
252
+ "non_embedding_paper_N": 475851904,
253
+ "total_trainable": 576059264
254
+ },
255
+ "pipeline_parallel_size": 1,
256
+ "planned_training_tokens": 7000031232,
257
+ "projection_dtype": "bfloat16 (production)",
258
+ "qk_norm": "functional_rmsnorm_after_rope",
259
+ "residual_and_norm_scale_dtype": "float32",
260
+ "residual_multiplier": 1.0,
261
+ "rope_base": 10000.0,
262
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
263
+ "sequence_length": 2048,
264
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
265
+ "tensor_count": 130,
266
+ "tensor_dtypes": {
267
+ "torch.bfloat16": 87,
268
+ "torch.float32": 43
269
+ },
270
+ "tensor_parallel_size": 1,
271
+ "tied_embeddings": false,
272
+ "tokenizer": {
273
+ "active_vocab_size": 32001,
274
+ "additional_token": "<|assistant_end|>",
275
+ "repo_id": "NousResearch/Llama-2-7b-hf",
276
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
277
+ },
278
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
279
+ "total_parameters": 576059264,
280
+ "total_trainable_parameters": 576059264,
281
+ "train_passes": 3,
282
+ "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
283
+ "training_code_dirty": false,
284
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
285
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
286
+ "training_tokens": 2097152000,
287
+ "vocab_size": 32001,
288
+ "width_scale": 22
289
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/launch_config.env ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
2
+ # See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
3
+ # See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
4
+ # a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
5
+ # Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
6
+
7
+ TRAIN_ARCHITECTURE=full_bandwidth
8
+ BACKBONE_PROFILE=paper2604_vanilla
9
+ TRAINING_RECIPE=paper2604_21106
10
+ RECIPE_OVERRIDES='{}'
11
+ # Use the installed Transformers release; the cluster's dev checkout is incompatible.
12
+ TRANSFORMERS_PATH=''
13
+ RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
14
+ RESUME_POLICY=fresh
15
+ REQUIRE_CHECKPOINT=false
16
+
17
+ # Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
18
+ # The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
19
+ DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
20
+ ONLINE_DATA=true
21
+ # Match the completed vanilla run: use the complete local raw cache when available.
22
+ ONLINE_SOURCE_MODE=auto
23
+ ONLINE_DATASET=HuggingFaceFW/fineweb-edu
24
+ ONLINE_DATASET_SUBSET=sample-100BT
25
+ ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
26
+ ONLINE_SOURCE_SPLIT=train
27
+ ONLINE_TEXT_COLUMN=text
28
+ ONLINE_VALIDATION_FRACTION=0.001
29
+ ONLINE_SPLIT_SEED=260421106
30
+ TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
31
+ ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
32
+ ONLINE_TOKENIZER_BATCH_SIZE=256
33
+ ONLINE_SHUFFLE_BUFFER_RECORDS=128
34
+ NUM_WORKERS=0
35
+ DATA_PREFLIGHT=false
36
+ EXPECTED_RECORD_LENGTH=2049
37
+ EXPECTED_INPUT_LENGTH=2048
38
+ MIN_MODEL_INPUT_TOKENS=7000000000
39
+
40
+ # Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
41
+ # The multi-node launcher supplies the node rank; preflight can override NNODES.
42
+ NNODES=4
43
+ GPUS_PER_NODE=8
44
+ TP=1
45
+ PP=1
46
+ # Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
47
+ MICRO_BATCH_SIZE=4
48
+ GLOBAL_BATCH_SIZE=128
49
+ SEQ_LENGTH=2048
50
+ MAX_POSITION_EMBEDDINGS=2048
51
+ # Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
52
+ # Use the vanilla LR curve from step 0 through the same 7B endpoint.
53
+ TRAIN_ITERS=26703
54
+
55
+ NUM_LAYERS=20
56
+ HIDDEN_SIZE=1408
57
+ FFN_HIDDEN_SIZE=5632
58
+ NUM_ATTENTION_HEADS=11
59
+ NUM_QUERY_GROUPS=11
60
+ GROUP_QUERY_ATTENTION=false
61
+ ATTENTION_OUTPUT_GATE=false
62
+ QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
63
+ SWIGLU=false # Profile supplies squared-ReLU.
64
+ DISABLE_BIAS_LINEAR=true
65
+ NORMALIZATION=RMSNorm
66
+ LAYERNORM_EPSILON=0.000001
67
+ HIDDEN_DROPOUT=0
68
+ ATTENTION_DROPOUT=0
69
+ WINDOW_SIZE=none
70
+ WINDOW_ATTN_SKIP_FREQ=1
71
+ POSITION_EMBEDDING_TYPE=rope
72
+ ROTARY_BASE=10000
73
+ UNTIE_EMBEDDINGS=true
74
+ TOKENIZER_TYPE=HuggingFaceTokenizer
75
+ VOCAB_SIZE=32001
76
+ MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
77
+ PRECISION=bf16
78
+ ENABLE_RECOMPUTE=false
79
+ MEGATRON_EXTRA_ARGS=''
80
+
81
+ # Use K=3 on every update, from update 1 through update 26,703.
82
+ FEEDBACK_PASS_MIXTURE=3:1.0
83
+ FEEDBACK_PROGRESSIVE_STAGES='[]'
84
+ FEEDBACK_EVAL_PASSES=3
85
+ # 1-based block outputs, with the final layer after trunk normalization.
86
+ # Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
87
+ FEEDBACK_LAYERS=2,11,20
88
+ FEEDBACK_LOSS_WEIGHT=1.0
89
+ FEEDBACK_JITTER_SIGMA=0
90
+ FEEDBACK_PREFIX_MIXIN=false
91
+ FEEDBACK_Z_LOSS_COEFFICIENT=0
92
+ FEEDBACK_LOG_HIDDEN_CHANGE=false
93
+
94
+ # Frequent logs and a short sampled evaluation for the initial training check.
95
+ WANDB_ENTITY=LoopTF-4-CSPs
96
+ WANDB_PROJECT=loopTF-workspace
97
+ WANDB_MODE=online
98
+ WANDB_RESUME=never
99
+ WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
100
+ WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
101
+ SAVE_INTERVAL=4000
102
+ EVAL_AT_START=true
103
+ EVAL_INTERVAL=100
104
+ EVAL_ITERS=2
105
+ LOG_INTERVAL=1
106
+ CKPT_FORMAT=torch
107
+ SEED=42
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/manifest.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_iteration": 8000,
3
+ "files": {
4
+ "README.md": {
5
+ "sha256": "aab5de6678fc9e11bc6f70fee55ae08a3f7dd392815042ddf025b0f3ba6b41a7",
6
+ "size": 2696
7
+ },
8
+ "config.json": {
9
+ "sha256": "5ef6a6b687625bc4d8ebabb0eef5da14e72a9133cc48317b27fe086fb945cee6",
10
+ "size": 12271
11
+ },
12
+ "launch_config.env": {
13
+ "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
14
+ "size": 3606
15
+ },
16
+ "model.safetensors": {
17
+ "sha256": "c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5",
18
+ "size": 1152254688
19
+ },
20
+ "training_recipe.json": {
21
+ "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
22
+ "size": 4117
23
+ }
24
+ },
25
+ "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
26
+ "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_8000",
27
+ "repo_id": "Siyuc/loopTF-workspace",
28
+ "repo_type": "model",
29
+ "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
30
+ }
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5
3
+ size 1152254688
FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/training_recipe.json ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 576059264,
120
+ "feedback_parameters": 3967744,
121
+ "feedback_layer_projection_parameters": 5947392
122
+ },
123
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
124
+ "execution_architecture": "full_bandwidth",
125
+ "objective": "latent_feedback_causal_lm",
126
+ "feedback_enabled": true,
127
+ "train_passes": 3,
128
+ "eval_passes": 3,
129
+ "feedback_pass_mixture": {
130
+ "3": 1.0
131
+ },
132
+ "feedback_progressive_stages": [],
133
+ "feedback_layers": [
134
+ 2,
135
+ 11,
136
+ 20
137
+ ],
138
+ "feedback_layer_fusion": "concatenate_then_bias_free_linear",
139
+ "feedback_layer_projection_init": "final_layer_identity",
140
+ "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
141
+ }
142
+ }
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/NAMING.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "wandb_id": "qwbqyrvt",
3
+ "previous_wandb_name": "r4-parent-fbw-b16--qwbqyrvt",
4
+ "training_run_name": "r4_parent_fbw_b16_20260920-101859",
5
+ "canonical_name": "FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt",
6
+ "architecture": "full_bandwidth",
7
+ "hf_old_paths": [
8
+ "r4_parent_fbw_b16_20260920-101859"
9
+ ]
10
+ }
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/README.md ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # r4_parent_fbw_b16_20260920-101859 / step_12000
2
+
3
+ Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
4
+
5
+ Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
6
+
7
+ Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
8
+
9
+ ## Native CPU loading
10
+
11
+ ```python
12
+ import json, torch, importlib, tempfile
13
+ from argparse import Namespace
14
+ from megatron.core import parallel_state
15
+ from megatron.core.process_groups_config import ProcessGroupCollection
16
+ from full_bandwidth_tf.megatron.config import build_model_config
17
+ from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
18
+ from safetensors.torch import load_file
19
+
20
+ def decode(x):
21
+ if isinstance(x, dict):
22
+ if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
23
+ if "__dataclass__" in x:
24
+ module, cls, values = x["__dataclass__"]
25
+ return getattr(importlib.import_module(module), cls)(**decode(values))
26
+ if "__enum__" in x:
27
+ module, cls, member = x["__enum__"]
28
+ return getattr(importlib.import_module(module), cls)[member]
29
+ return {k: decode(v) for k, v in x.items()}
30
+ if isinstance(x, list): return [decode(v) for v in x]
31
+ return x
32
+
33
+ c = json.load(open("config.json"))
34
+ a = Namespace(**decode(c["model_arguments"]))
35
+ a.use_cpu_initialization = True
36
+ a.perform_initialization = False
37
+ cfg = build_model_config(a)
38
+ # Standalone CPU loading; use your existing groups in an initialized runtime.
39
+ with tempfile.TemporaryDirectory() as tmp:
40
+ torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
41
+ parallel_state.initialize_model_parallel()
42
+ with torch.device("meta"):
43
+ model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
44
+ state = load_file("model.safetensors") | c["non_tensor_state"]
45
+ model.load_state_dict(state, strict=True, assign=True)
46
+ model.eval()
47
+ torch.distributed.destroy_process_group()
48
+ ```
49
+
50
+ All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/config.json ADDED
@@ -0,0 +1,1339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint_format": "safetensors_native_megatron_state_dict",
3
+ "architecture": "full_bandwidth",
4
+ "backbone_profile": "paper2604_vanilla",
5
+ "run_name": "r4_parent_fbw_b16_20260920-101859",
6
+ "iteration": 12000,
7
+ "checkpoint_version": 3.0,
8
+ "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
9
+ "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
10
+ "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
11
+ "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
12
+ "parameter_counts": {
13
+ "total": 570111872,
14
+ "trainable": 570111872
15
+ },
16
+ "model_arguments": {
17
+ "tensor_model_parallel_size": 1,
18
+ "tensor_parallel_num_weight_shards": 1,
19
+ "pipeline_model_parallel_comm_backend": null,
20
+ "pipeline_model_parallel_size": 1,
21
+ "sequence_parallel": false,
22
+ "context_parallel_size": 1,
23
+ "hierarchical_context_parallel_sizes": null,
24
+ "expert_model_parallel_size": 1,
25
+ "expert_tensor_parallel_size": 1,
26
+ "expert_tensor_parallel_num_weight_shards": 1,
27
+ "perform_initialization": true,
28
+ "use_cpu_initialization": null,
29
+ "fp16": false,
30
+ "bf16": true,
31
+ "deterministic_mode": false,
32
+ "tp_comm_overlap": false,
33
+ "tp_comm_bulk_wgrad": true,
34
+ "tp_comm_bulk_dgrad": true,
35
+ "tp_comm_overlap_ag": true,
36
+ "tp_comm_overlap_rs": true,
37
+ "tp_comm_overlap_rs_dgrad": false,
38
+ "tp_comm_split_ag": true,
39
+ "tp_comm_split_rs": true,
40
+ "cross_entropy_loss_fusion": false,
41
+ "cross_entropy_fusion_impl": "native",
42
+ "tp_comm_bootstrap_backend": "nccl",
43
+ "overlap_moe_expert_parallel_comm": false,
44
+ "delay_wgrad_compute": false,
45
+ "overlap_dispatch_backward_with_experts_wgrad": false,
46
+ "ep_overlap_early_attn_memory_release": false,
47
+ "use_ring_exchange_p2p": false,
48
+ "defer_embedding_wgrad_compute": false,
49
+ "wgrad_deferral_limit": 0,
50
+ "overlap_p2p_comm_warmup_flush": false,
51
+ "microbatch_group_size_per_vp_stage": null,
52
+ "mtp_standalone": false,
53
+ "cpu_offloading_num_layers": 0,
54
+ "cpu_offloading_retain_pinned_cpu_buffers": false,
55
+ "num_layers": 20,
56
+ "mtp_num_layers": null,
57
+ "mtp_loss_scaling_factor": 0.1,
58
+ "mtp_use_repeated_layer": false,
59
+ "mtp_detach_heads": false,
60
+ "mtp_hsm": false,
61
+ "mtp_hybrid_override_pattern": null,
62
+ "account_for_embedding_in_pipeline_split": false,
63
+ "account_for_loss_in_pipeline_split": false,
64
+ "hidden_size": 1408,
65
+ "num_attention_heads": 11,
66
+ "attention_backend": {
67
+ "__enum__": [
68
+ "megatron.core.transformer.enums",
69
+ "AttnBackend",
70
+ "auto"
71
+ ]
72
+ },
73
+ "flash_attention_version": null,
74
+ "softmax_type": "vanilla",
75
+ "num_query_groups": 11,
76
+ "ffn_hidden_size": 5632,
77
+ "kv_channels": 128,
78
+ "hidden_dropout": 0.0,
79
+ "attention_dropout": 0.0,
80
+ "fp32_residual_connection": false,
81
+ "apply_residual_connection_post_layernorm": false,
82
+ "layernorm_epsilon": 1e-06,
83
+ "layernorm_zero_centered_gamma": false,
84
+ "add_bias_linear": false,
85
+ "add_qkv_bias": false,
86
+ "glu_linear_offset": 0.0,
87
+ "activation_func_clamp_value": null,
88
+ "rotary_interleaved": false,
89
+ "normalization": "RMSNorm",
90
+ "qk_layernorm": false,
91
+ "qk_l2_norm": false,
92
+ "qk_clip": false,
93
+ "qk_clip_alpha": 0.5,
94
+ "qk_clip_threshold": 100,
95
+ "attention_output_gate": false,
96
+ "calculate_per_token_loss": false,
97
+ "multi_latent_attention": false,
98
+ "experimental_attention_variant": null,
99
+ "dsa_indexer_n_heads": null,
100
+ "dsa_indexer_head_dim": null,
101
+ "dsa_indexer_topk": null,
102
+ "dsa_indexer_topk_freq": 1,
103
+ "dsa_indexer_skip_topk_offset": 0,
104
+ "dsa_indexer_loss_coeff": null,
105
+ "dsa_indexer_use_sparse_loss": false,
106
+ "dsa_kernel_backend": "none",
107
+ "dsa_indexer_rope_interleaved": false,
108
+ "dsa_indexer_rotate_activation": true,
109
+ "dsa_indexer_scoring_relu": true,
110
+ "dsa_indexer_k_norm_epsilon": null,
111
+ "dsa_indexer_k_norm_fp32": false,
112
+ "csa_window_size": 128,
113
+ "csa_compress_ratios": null,
114
+ "csa_compress_rotary_base": 40000.0,
115
+ "csa_dense_mode": false,
116
+ "linear_conv_kernel_dim": 4,
117
+ "linear_key_head_dim": 128,
118
+ "linear_value_head_dim": 128,
119
+ "linear_num_key_heads": 16,
120
+ "linear_num_value_heads": 32,
121
+ "init_method_std": 0.02,
122
+ "embedding_init_method_std": null,
123
+ "init_model_with_meta_device": false,
124
+ "use_mup": false,
125
+ "mup_width_mult": 1.0,
126
+ "mup_base_hidden_size": null,
127
+ "mup_embedding_mult": 1.0,
128
+ "mup_output_mult": 1.0,
129
+ "mup_base_head_dim": null,
130
+ "mup_attn_scale_power": 1.0,
131
+ "apply_query_key_layer_scaling": false,
132
+ "disable_bf16_reduced_precision_matmul": false,
133
+ "use_fused_weighted_squared_relu": false,
134
+ "fused_residual_rmsnorm": false,
135
+ "use_transformer_engine_op_fuser": false,
136
+ "recompute_granularity": null,
137
+ "recompute_method": null,
138
+ "recompute_num_layers": null,
139
+ "distribute_saved_activations": false,
140
+ "recompute_modules": null,
141
+ "fp8": null,
142
+ "fp8_recipe": "delayed",
143
+ "fp8_quantizer_factory": null,
144
+ "fp8_margin": 0,
145
+ "fp8_interval": 1,
146
+ "fp8_amax_history_len": 1,
147
+ "fp8_amax_compute_algo": "most_recent",
148
+ "fp8_wgrad": true,
149
+ "fp8_output_proj": false,
150
+ "first_last_layers_bf16": false,
151
+ "num_layers_at_start_in_bf16": 1,
152
+ "num_layers_at_end_in_bf16": 1,
153
+ "use_kitchen_attention": false,
154
+ "kitchen_attention_backend": "sdpa",
155
+ "fp4": null,
156
+ "fp4_recipe": "nvfp4",
157
+ "fp4_quantizer_factory": null,
158
+ "high_priority_a2a_comm_stream": false,
159
+ "moe_shared_expert_intermediate_size": null,
160
+ "moe_shared_expert_gate": false,
161
+ "moe_shared_expert_overlap": false,
162
+ "use_grouped_gemm_for_shared_expert": false,
163
+ "moe_shared_expert_glu_interleave_size": null,
164
+ "moe_ffn_hidden_size": null,
165
+ "moe_router_topk": 2,
166
+ "moe_enable_routing_replay": false,
167
+ "moe_router_padding_for_quantization": false,
168
+ "moe_router_padding_for_fp8": false,
169
+ "moe_router_num_groups": null,
170
+ "moe_router_group_topk": null,
171
+ "moe_router_pre_softmax": false,
172
+ "moe_router_topk_scaling_factor": null,
173
+ "moe_router_score_function": "softmax",
174
+ "moe_router_dtype": null,
175
+ "moe_router_skip_muon": false,
176
+ "moe_router_enable_expert_bias": false,
177
+ "moe_router_bias_update_rate": 0.001,
178
+ "moe_router_quantile_balancing_ema": 0.0,
179
+ "moe_router_force_load_balancing": false,
180
+ "moe_router_force_biased": null,
181
+ "use_grouped_gemm_for_dense_mlp": false,
182
+ "moe_grouped_gemm": false,
183
+ "moe_use_grouped_tensor": false,
184
+ "moe_single_grouped_weight": false,
185
+ "moe_single_grouped_bias": false,
186
+ "moe_z_loss_coeff": null,
187
+ "moe_input_jitter_eps": null,
188
+ "moe_token_dispatcher_type": "allgather",
189
+ "moe_enable_deepep": false,
190
+ "moe_flex_dispatcher_backend": "deepep",
191
+ "moe_permute_fusion_into_hybridep": false,
192
+ "moe_hybridep_pad_uneven_dispatch_inputs": false,
193
+ "moe_per_layer_logging": false,
194
+ "moe_expert_capacity_factor": null,
195
+ "moe_pad_expert_input_to_capacity": false,
196
+ "moe_pad_experts_for_cuda_graph_inference": false,
197
+ "moe_token_drop_policy": "probs",
198
+ "moe_layer_recompute": false,
199
+ "moe_permute_fusion": false,
200
+ "moe_router_fusion": false,
201
+ "moe_apply_probs_on_input": false,
202
+ "moe_latent_size": null,
203
+ "moe_use_norm_before_up_proj": false,
204
+ "gtp_remat_opt_in_modules": [],
205
+ "moe_flex_dispatcher_num_sms": null,
206
+ "moe_deepep_num_sms": null,
207
+ "moe_hybridep_num_sms": null,
208
+ "moe_hybridep_num_blocks_permute": null,
209
+ "moe_hybridep_num_blocks_unpermute": null,
210
+ "moe_hybridep_num_sms_preprocessing": 108,
211
+ "moe_ncclep_zero_copy": false,
212
+ "moe_dispatch_fwd_dtype": "bf16",
213
+ "moe_combine_bwd_dtype": "bf16",
214
+ "moe_mlp_glu_interleave_size": null,
215
+ "moe_expert_rank_capacity_factor": null,
216
+ "linear_cp_mode": "headwise",
217
+ "linear_cp_layout": "zigzag",
218
+ "attention_cp_layout": "zigzag",
219
+ "enable_cuda_graph": false,
220
+ "cuda_graph_warmup_steps": 3,
221
+ "external_cuda_graph": false,
222
+ "cuda_graph_impl": "none",
223
+ "inference_cuda_graph_scope": {
224
+ "__enum__": [
225
+ "megatron.core.transformer.enums",
226
+ "InferenceCudaGraphScope",
227
+ "none"
228
+ ]
229
+ },
230
+ "enable_mhc_connections": false,
231
+ "mhc_num_residual_streams": 4,
232
+ "mhc_sinkhorn_iterations": 20,
233
+ "mhc_init_gating_factor": 0.01,
234
+ "use_fused_mhc": false,
235
+ "mhc_fused_backend": "auto",
236
+ "mhc_recompute_layer_num": null,
237
+ "clone_scatter_output_in_embedding": true,
238
+ "config_logger_dir": "",
239
+ "flash_decode": false,
240
+ "batch_invariant_mode": false,
241
+ "batch_invariant_backend": "te_native",
242
+ "batch_invariant_collective": "ordered",
243
+ "use_te_activation_func": false,
244
+ "symmetric_ar_type": null,
245
+ "nccl_all_reduce_for_prefill": false,
246
+ "inference_fuse_tp_communication": false,
247
+ "inference_disable_triton_nvls_kernels": false,
248
+ "inference_grouped_gemm_backend": "vllm",
249
+ "inference_moe_disable_fused_quant_kernels": false,
250
+ "inference_flashinfer_mxfp8_token_capacity": null,
251
+ "inference_moe_token_dispatcher_type": "nvls",
252
+ "mrope_section": null,
253
+ "is_hybrid_model": false,
254
+ "mamba_state_dim": 128,
255
+ "mamba_head_dim": 64,
256
+ "mamba_num_groups": 8,
257
+ "mamba_num_heads": null,
258
+ "gdp_num_householder": 3,
259
+ "use_mamba_mem_eff_path": true,
260
+ "gdp_cutedsl_kernel": false,
261
+ "gdp_num_chunk_states_to_recompute": 2,
262
+ "mlp_chunks_for_prefill": 1,
263
+ "mlp_chunks_for_training": 1,
264
+ "transformer_impl": "local",
265
+ "fine_grained_activation_offloading": false,
266
+ "offload_modules": [],
267
+ "min_offloaded_tensor_size": 1048576,
268
+ "delay_offload_until_cuda_graph": false,
269
+ "delta_offload_bytes_across_pp_ranks": 0,
270
+ "activation_offload_fraction": 1.0,
271
+ "moe_paged_stash": false,
272
+ "moe_paged_stash_page_size": 64,
273
+ "moe_paged_stash_buffer_size_factor_cuda": 1.1,
274
+ "moe_paged_stash_buffer_size_factor_cpu": 0.0,
275
+ "fine_grained_offloading_max_inflight_offloads": null,
276
+ "encoder_num_layers": 20,
277
+ "decoder_num_layers": null,
278
+ "group_query_attention": false,
279
+ "window_size": null,
280
+ "window_attn_skip_freq": null,
281
+ "max_position_embeddings": 2048,
282
+ "position_embedding_type": "rope",
283
+ "yarn_original_max_position_embeddings": null,
284
+ "yarn_beta_fast": null,
285
+ "yarn_beta_slow": null,
286
+ "yarn_correction_range_round_to_int": null,
287
+ "relative_attention_num_buckets": 32,
288
+ "relative_attention_max_distance": 128,
289
+ "use_rotary_position_embeddings": false,
290
+ "rotary_base": 10000,
291
+ "rotary_percent": 1.0,
292
+ "rotary_seq_len_interpolation_factor": null,
293
+ "use_rope_scaling": false,
294
+ "rope_scaling_factor": 8.0,
295
+ "no_rope_freq": null,
296
+ "add_position_embedding": true,
297
+ "make_vocab_size_divisible_by": 64,
298
+ "openai_gelu": false,
299
+ "squared_relu": false,
300
+ "swiglu": false,
301
+ "quick_geglu": false,
302
+ "onnx_safe": null,
303
+ "bert_binary_head": true,
304
+ "untie_embeddings_and_output_weights": true,
305
+ "weight_decay": 0.0,
306
+ "apply_wd_to_qk_layernorm": false,
307
+ "clip_grad": 0.0,
308
+ "adam_beta1": 0.8,
309
+ "adam_beta2": 0.95,
310
+ "adam_eps": 1e-10,
311
+ "sgd_momentum": 0.9,
312
+ "muon_momentum": 0.95,
313
+ "muon_split_qkv": true,
314
+ "muon_nesterov": true,
315
+ "muon_scale_mode": "spectral",
316
+ "muon_fp32_matmul_prec": "medium",
317
+ "muon_coefficient_type": "quintic",
318
+ "muon_num_ns_steps": 5,
319
+ "muon_tp_mode": "duplicated",
320
+ "muon_use_syrk": false,
321
+ "muon_extra_scale_factor": 1.0,
322
+ "muon_scalar_optimizer": "adam",
323
+ "lion_beta1": 0.95,
324
+ "lion_beta2": 0.98,
325
+ "no_weight_decay_cond_type": null,
326
+ "profile": false,
327
+ "profile_step_start": 10,
328
+ "profile_step_end": 12,
329
+ "use_pytorch_profiler": false,
330
+ "pytorch_profiler_collect_shapes": false,
331
+ "pytorch_profiler_collect_callstack": false,
332
+ "pytorch_profiler_collect_chakra": false,
333
+ "profile_ranks": [],
334
+ "record_memory_history": false,
335
+ "memory_snapshot_path": "snapshot.pickle",
336
+ "record_shapes": false,
337
+ "nvtx_ranges": false,
338
+ "micro_batch_size": 8,
339
+ "global_batch_size": 128,
340
+ "rampup_batch_size": null,
341
+ "step_batch_size_schedule": null,
342
+ "decrease_batch_size_if_needed": false,
343
+ "empty_unused_memory_level": 0,
344
+ "check_weight_hash_across_dp_replicas_interval": null,
345
+ "gpu_sniff_test_interval": null,
346
+ "train_sync_interval": null,
347
+ "train_iters": 26703,
348
+ "train_samples": null,
349
+ "exit_interval": 26703,
350
+ "exit_duration_in_mins": null,
351
+ "exit_signal_handler": false,
352
+ "exit_signal": {
353
+ "__enum__": [
354
+ "signal",
355
+ "Signals",
356
+ "SIGTERM"
357
+ ]
358
+ },
359
+ "exit_signal_handler_for_dataloader": false,
360
+ "exit_signal_handler_for_training": false,
361
+ "manual_gc": false,
362
+ "manual_gc_interval": 0,
363
+ "manual_gc_eval": true,
364
+ "iterations_to_skip": [],
365
+ "check_for_nan_in_loss_and_grad": true,
366
+ "check_for_large_grads": false,
367
+ "result_rejected_tracker_filename": null,
368
+ "tp_comm_overlap_cfg": null,
369
+ "masked_softmax_fusion": true,
370
+ "bias_gelu_fusion": false,
371
+ "bias_swiglu_fusion": true,
372
+ "bias_dropout_fusion": false,
373
+ "apply_rope_fusion": false,
374
+ "rope_type": null,
375
+ "use_flash_attn": false,
376
+ "optimizer": "muonh",
377
+ "optimizer_cpu_offload": false,
378
+ "optimizer_cuda_graph": false,
379
+ "optimizer_offload_fraction": 1.0,
380
+ "use_torch_optimizer_for_cpu_offload": false,
381
+ "overlap_cpu_optimizer_d2h_h2d": false,
382
+ "dump_param_to_param_group_map": null,
383
+ "pin_cpu_grads": true,
384
+ "pin_cpu_params": true,
385
+ "dataloader_type": "external",
386
+ "no_persist_layer_norm": false,
387
+ "gradient_accumulation_fusion": false,
388
+ "deprecated_use_mcore_models": false,
389
+ "perform_rl_step": false,
390
+ "rl_prompts_per_eval": 32,
391
+ "grpo_prompts_per_step": 32,
392
+ "grpo_group_size": 2,
393
+ "rl_generation_lag": null,
394
+ "rl_max_inflight_requests": null,
395
+ "rl_submission_granularity": "B",
396
+ "rl_consumption_granularity": "B",
397
+ "rl_durable_rollout_bank": false,
398
+ "rl_rollout_bank_dir": null,
399
+ "rl_rollout_bank_max_bytes": 0,
400
+ "grpo_iterations": 2,
401
+ "grpo_clamp_eps_lower": 0.01,
402
+ "grpo_clamp_eps_upper": 0.01,
403
+ "grpo_kl_beta": 0.001,
404
+ "grpo_entropy_term_weight": 0.0,
405
+ "grpo_filter_groups_with_same_reward": false,
406
+ "langrl_env_config": null,
407
+ "rl_default_temperature": 1.0,
408
+ "rl_default_top_p": 0,
409
+ "rl_default_top_k": -1,
410
+ "rl_offload_optimizer_during_inference": false,
411
+ "rl_kv_cache_management_mode": "persist",
412
+ "rl_persist_cuda_graphs": false,
413
+ "rl_partial_rollouts": false,
414
+ "rl_inference_logprobs_is_correction": false,
415
+ "rl_importance_sampling_truncation_coef": null,
416
+ "rl_use_sequence_packing": false,
417
+ "rl_sequence_packing_max_sequences_per_bin": 50,
418
+ "rl_sequence_packing_algo": "fifo",
419
+ "rl_training_cuda_graphs": false,
420
+ "rl_inference_tensor_model_parallel_size": null,
421
+ "rl_inference_pipeline_model_parallel_size": null,
422
+ "rl_inference_expert_model_parallel_size": null,
423
+ "rl_inference_expert_tensor_model_parallel_size": null,
424
+ "rl_inference_model_unified_memory_level": 0,
425
+ "rl_offload_inference_model_weights_when_idle": false,
426
+ "refit_method": "gloo",
427
+ "refit_execution_batch_bytes": null,
428
+ "rl_verify_model_weights_swap": false,
429
+ "rl_skip_bos_token": false,
430
+ "rl_profile": false,
431
+ "rl_profile_dir": null,
432
+ "rl_inference_parsers": [],
433
+ "seed": 42,
434
+ "te_rng_tracker": false,
435
+ "inference_rng_tracker": false,
436
+ "data_parallel_random_init": false,
437
+ "init_method_xavier_uniform": false,
438
+ "lr_decay_style": "linear",
439
+ "lr_wsd_decay_style": "linear",
440
+ "lr_decay_iters": null,
441
+ "lr_decay_samples": null,
442
+ "lr_wsd_decay_iters": 0,
443
+ "lr_wsd_decay_samples": null,
444
+ "lr_warmup_fraction": null,
445
+ "lr_warmup_iters": 0,
446
+ "lr_warmup_samples": 0,
447
+ "lr_warmup_init": 0.0,
448
+ "override_opt_param_scheduler": false,
449
+ "use_checkpoint_opt_param_scheduler": false,
450
+ "start_weight_decay": 0.0,
451
+ "end_weight_decay": 0.0,
452
+ "weight_decay_incr_style": "constant",
453
+ "lr": 0.007470638533717786,
454
+ "min_lr": 0.0007470638533717787,
455
+ "decoupled_lr": null,
456
+ "decoupled_min_lr": null,
457
+ "freeze_all_layers": false,
458
+ "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
459
+ "save_interval": 4000,
460
+ "save_params_interval": null,
461
+ "save_activations_interval": null,
462
+ "save_tokens_per_expert_interval": null,
463
+ "save_wgrads_interval": null,
464
+ "save_dgrads_interval": null,
465
+ "save_retain_interval": 4000,
466
+ "load": null,
467
+ "load_main_params_from_ckpt": false,
468
+ "non_persistent_save_interval": null,
469
+ "non_persistent_ckpt_type": null,
470
+ "non_persistent_global_ckpt_dir": null,
471
+ "non_persistent_local_ckpt_dir": null,
472
+ "non_persistent_local_ckpt_algo": "fully_parallel",
473
+ "finetune": false,
474
+ "pretrained_checkpoint": null,
475
+ "ckpt_step": null,
476
+ "use_checkpoint_args": false,
477
+ "use_mp_args_from_checkpoint_args": false,
478
+ "use_tokenizer_model_from_checkpoint_args": true,
479
+ "exit_on_missing_checkpoint": false,
480
+ "ckpt_format": "torch",
481
+ "auto_detect_ckpt_format": false,
482
+ "ckpt_convert_format": null,
483
+ "ckpt_convert_save": null,
484
+ "ckpt_convert_update_legacy_dist_opt_format": false,
485
+ "ckpt_fully_parallel_save": true,
486
+ "async_save": false,
487
+ "async_strategy": "mcore",
488
+ "use_persistent_ckpt_worker": false,
489
+ "async_ckpt_cpu_priority": 10,
490
+ "async_ckpt_io_priority": 3,
491
+ "async_ckpt_use_cpu_shm": false,
492
+ "ckpt_fully_parallel_load": false,
493
+ "ckpt_fully_parallel_load_exchange_algo": "broadcast",
494
+ "ckpt_fully_parallel_load_per_rank_objects": false,
495
+ "ckpt_fully_parallel_save_process_group": "dp",
496
+ "ckpt_fully_parallel_load_process_group": "dp",
497
+ "ckpt_assume_constant_structure": false,
498
+ "ckpt_pg_tensors_cache_path": null,
499
+ "ckpt_pg_tensors_cache_create": false,
500
+ "ckpt_load_validate_sharding_integrity": true,
501
+ "strict_fsdp_dtensor_load": true,
502
+ "dist_ckpt_strictness": "assume_ok_unexpected",
503
+ "dist_ckpt_save_pre_mcore_014": false,
504
+ "dist_ckpt_optim_fully_reshardable": false,
505
+ "distrib_optim_fully_reshardable_mem_efficient": false,
506
+ "save_tokenizer_assets": true,
507
+ "replication": false,
508
+ "replication_jump": null,
509
+ "replication_factor": 2,
510
+ "verify_integrity": false,
511
+ "no_save_optim": null,
512
+ "no_save_rng": null,
513
+ "no_load_optim": null,
514
+ "no_load_rng": null,
515
+ "override_ckpt_iteration": null,
516
+ "use_dist_ckpt_deprecated": false,
517
+ "dist_ckpt_format_deprecated": null,
518
+ "dist_ckpt_workers": 1,
519
+ "ckpt_fully_parallel_save_deprecated": false,
520
+ "ckpt_drop_redundant_extra_state": false,
521
+ "grad_reduce_in_bf16": false,
522
+ "loss_scale": null,
523
+ "initial_loss_scale": 4294967296,
524
+ "min_loss_scale": 1.0,
525
+ "loss_scale_window": 1000,
526
+ "hysteresis": 2,
527
+ "attention_softmax_in_fp32": false,
528
+ "accumulate_allreduce_grads_in_fp32": true,
529
+ "fp16_lm_cross_entropy": false,
530
+ "logit_dtype": null,
531
+ "reuse_grad_buf_for_mxfp8_param_ag": false,
532
+ "mamba_training_ssm_states_dtype": null,
533
+ "distributed_backend": "nccl",
534
+ "distributed_timeout_minutes": 10,
535
+ "align_grad_reduce": true,
536
+ "local_rank": 0,
537
+ "lazy_mpu_init": false,
538
+ "use_megatron_fsdp": false,
539
+ "use_torch_fsdp2": false,
540
+ "nccl_communicator_config_path": null,
541
+ "use_tp_pp_dp_mapping": false,
542
+ "use_gloo_process_groups": true,
543
+ "use_sharp": false,
544
+ "sharp_enabled_group": null,
545
+ "high_priority_stream_groups": [],
546
+ "distributed_timeout_seconds_after_init": null,
547
+ "flight_recorder_dump_path": null,
548
+ "flight_recorder_trace_buffer_size": 2000,
549
+ "flight_recorder_dump_on_timeout": true,
550
+ "flight_recorder_include_stack_trace": false,
551
+ "flight_recorder_include_only_active": true,
552
+ "flight_recorder_extra_dump_on_exec": true,
553
+ "disable_jit_fuser": false,
554
+ "decoder_first_pipeline_num_layers": null,
555
+ "decoder_last_pipeline_num_layers": null,
556
+ "pipeline_model_parallel_layout": null,
557
+ "num_layers_per_virtual_pipeline_stage": null,
558
+ "num_virtual_stages_per_pipeline_rank": null,
559
+ "overlap_p2p_comm": false,
560
+ "overlap_grad_reduce": false,
561
+ "ddp_num_buckets": null,
562
+ "ddp_bucket_size": null,
563
+ "ddp_pad_buckets_for_high_nccl_busbw": false,
564
+ "ddp_reduce_scatter_with_fp32_accumulation": false,
565
+ "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
566
+ "ddp_param_name_patterns_for_fp32_local_accumulation": [],
567
+ "ddp_average_in_collective": false,
568
+ "overlap_param_gather": false,
569
+ "overlap_param_gather_with_optimizer_step": false,
570
+ "align_param_gather": false,
571
+ "use_distributed_optimizer": false,
572
+ "megatron_fsdp_version": 1,
573
+ "use_layer_wise_param_layout": true,
574
+ "nccl_ub": false,
575
+ "disable_symmetric_registration": false,
576
+ "gtp_remat_nccl_ub": false,
577
+ "gtp_expert_remat_nccl_ub": false,
578
+ "fsdp_manual_registration": false,
579
+ "create_all_gather_group": false,
580
+ "data_parallel_sharding_strategy": "optim_grads_params",
581
+ "expert_data_parallel_sharding_strategy": null,
582
+ "outer_dp_sharding_strategy": "no_shard",
583
+ "hfsdp_param_gather_overlap": false,
584
+ "gradient_reduce_div_fusion": true,
585
+ "fsdp_double_buffer": false,
586
+ "suggested_communication_unit_size": null,
587
+ "keep_fp8_transpose_cache": false,
588
+ "enable_full_sharding_in_hsdp": false,
589
+ "num_distributed_optimizer_instances": 1,
590
+ "torch_fsdp2_reshard_after_forward": true,
591
+ "cp_comm_type": [
592
+ "p2p"
593
+ ],
594
+ "max_seqlen_per_dp_cp_rank": null,
595
+ "hybrid_context_parallel": false,
596
+ "sequence_packing_scheduler": null,
597
+ "fake_process_group": false,
598
+ "eval_iters": 2,
599
+ "eval_interval": 100,
600
+ "start_eval_at_iter": null,
601
+ "eval_global_batch_size": 128,
602
+ "eval_micro_batch_size": 8,
603
+ "skip_train": false,
604
+ "test_mode": false,
605
+ "full_validation": false,
606
+ "multiple_validation_sets": false,
607
+ "validation_set_names": null,
608
+ "data_path": null,
609
+ "phase_transition_iterations": null,
610
+ "split": null,
611
+ "train_data_path": null,
612
+ "valid_data_path": null,
613
+ "test_data_path": null,
614
+ "data_args_path": null,
615
+ "per_split_data_args_path": null,
616
+ "per_dataset_sequences_path": null,
617
+ "dataloader_fast_cache_load": false,
618
+ "dataloader_defer_npy_index_mmap": false,
619
+ "data_cache_path": null,
620
+ "mmap_bin_files": true,
621
+ "mock_data": false,
622
+ "seq_length": 2048,
623
+ "encoder_seq_length": 2048,
624
+ "decoder_seq_length": null,
625
+ "sample_rate": 1.0,
626
+ "mask_prob": 0.15,
627
+ "short_seq_prob": 0.1,
628
+ "num_workers": 0,
629
+ "reset_position_ids": false,
630
+ "reset_attention_mask": false,
631
+ "dataloader_inter_document_masking": false,
632
+ "eod_mask_loss": false,
633
+ "create_attention_mask_in_dataloader": false,
634
+ "num_dataset_builder_threads": 1,
635
+ "object_storage_cache_path": null,
636
+ "mid_level_dataset_surplus": 0.005,
637
+ "allow_ambiguous_pad_tokens": false,
638
+ "fim_data": false,
639
+ "fim_rate": 0.5,
640
+ "fim_spm_rate": 0.5,
641
+ "fim_split_sample": null,
642
+ "fim_fragment_rate": null,
643
+ "fim_no_prefix": null,
644
+ "fim_prefix_token": "<fim_prefix>",
645
+ "fim_middle_token": "<fim_middle>",
646
+ "fim_suffix_token": "<fim_suffix>",
647
+ "fim_pad_token": "<fim_pad>",
648
+ "fim_eod_token": "<|endoftext|>",
649
+ "vocab_size": 32001,
650
+ "padded_vocab_size": 32064,
651
+ "pad_vocab_size": true,
652
+ "vocab_file": null,
653
+ "merge_file": null,
654
+ "vocab_extra_ids": 0,
655
+ "tokenizer_type": "HuggingFaceTokenizer",
656
+ "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
657
+ "metadata_path": null,
658
+ "special_tokens": null,
659
+ "tiktoken_pattern": null,
660
+ "tiktoken_num_special_tokens": 1000,
661
+ "tokenizer_sentencepiece_legacy": false,
662
+ "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
663
+ "tokenizer_hf_no_use_fast": false,
664
+ "tokenizer_hf_no_include_special_tokens": false,
665
+ "trust_remote_code": false,
666
+ "null_tokenizer_eod_id": null,
667
+ "null_tokenizer_pad_id": -1,
668
+ "chat_template": null,
669
+ "use_gigatoken": false,
670
+ "adlr_autoresume": false,
671
+ "adlr_autoresume_interval": 1000,
672
+ "ict_head_size": null,
673
+ "biencoder_projection_dim": 0,
674
+ "biencoder_shared_query_context_model": false,
675
+ "ict_load": null,
676
+ "bert_load": null,
677
+ "titles_data_path": null,
678
+ "query_in_block_prob": 0.1,
679
+ "use_one_sent_docs": false,
680
+ "evidence_data_path": null,
681
+ "retriever_report_topk_accuracies": [],
682
+ "retriever_score_scaling": false,
683
+ "block_data_path": null,
684
+ "embedding_path": null,
685
+ "indexer_batch_size": 128,
686
+ "indexer_log_interval": 1000,
687
+ "num_classes": 1000,
688
+ "img_h": 224,
689
+ "img_w": 224,
690
+ "num_channels": 3,
691
+ "patch_dim": 16,
692
+ "classes_fraction": 1.0,
693
+ "data_per_class_fraction": 1.0,
694
+ "data_sharding": true,
695
+ "head_lr_mult": 1.0,
696
+ "vision_pretraining": false,
697
+ "vision_pretraining_type": "classify",
698
+ "vision_backbone_type": "vit",
699
+ "swin_backbone_type": "tiny",
700
+ "mask_type": "random",
701
+ "mask_factor": 1.0,
702
+ "iter_per_epoch": 1250,
703
+ "dino_local_img_size": 96,
704
+ "dino_local_crops_number": 10,
705
+ "dino_head_hidden_size": 2048,
706
+ "dino_bottleneck_size": 256,
707
+ "dino_freeze_last_layer": 1,
708
+ "dino_norm_last_layer": false,
709
+ "dino_warmup_teacher_temp": 0.04,
710
+ "dino_teacher_temp": 0.07,
711
+ "dino_warmup_teacher_temp_epochs": 30,
712
+ "num_experts": null,
713
+ "moe_layer_freq": 1,
714
+ "moe_use_upcycling": false,
715
+ "moe_router_load_balancing_type": "aux_loss",
716
+ "moe_aux_loss_coeff": 0.0,
717
+ "moe_upcycling_granularity": 1,
718
+ "q_lora_rank": null,
719
+ "kv_lora_rank": 32,
720
+ "qk_head_dim": 128,
721
+ "qk_pos_emb_head_dim": 64,
722
+ "v_head_dim": 128,
723
+ "rotary_scaling_factor": 1.0,
724
+ "mscale": 1.0,
725
+ "mscale_all_dim": 0.0,
726
+ "cache_mla_latents": false,
727
+ "mla_down_proj_fusion": false,
728
+ "linear_attention_freq": null,
729
+ "heterogeneous_layers_config_path": null,
730
+ "heterogeneous_layers_config_encoded_json": null,
731
+ "log_interval": 1,
732
+ "log_params_norm": false,
733
+ "log_throughput": false,
734
+ "log_progress": false,
735
+ "timing_log_level": 0,
736
+ "timing_log_option": "minmax",
737
+ "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
738
+ "tensorboard_log_interval": 1,
739
+ "tensorboard_queue_size": 1000,
740
+ "log_timers_to_tensorboard": true,
741
+ "log_loss_scale_to_tensorboard": true,
742
+ "log_validation_ppl_to_tensorboard": false,
743
+ "log_memory_to_tensorboard": false,
744
+ "log_memory_interval": null,
745
+ "log_device_memory_used": false,
746
+ "log_num_zeros_in_grad": false,
747
+ "log_max_attention_logit": false,
748
+ "barrier_with_L1_time": true,
749
+ "log_world_size_to_tensorboard": false,
750
+ "wandb_project": "loopTF-workspace",
751
+ "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
752
+ "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
753
+ "wandb_entity": "LoopTF-4-CSPs",
754
+ "logging_level": null,
755
+ "log_energy": false,
756
+ "moe_routing_trace_path": null,
757
+ "moe_routing_trace_max_training_iters": null,
758
+ "moe_routing_trace_capture_logits": false,
759
+ "moe_routing_trace_capture_hidden_states": false,
760
+ "moe_routing_trace_dump_weights": false,
761
+ "otel_enabled": false,
762
+ "otel_service_name": null,
763
+ "otel_span_groups": null,
764
+ "logits_save_top_k": null,
765
+ "logits_save_top_p": null,
766
+ "logits_save_top_p_min_k": 1,
767
+ "logits_save_dir": null,
768
+ "logits_save_dtype": "fp16",
769
+ "logits_load_dir": null,
770
+ "logits_load_decode_threads": 4,
771
+ "logits_load_prefetch_factor": 3,
772
+ "logits_load_msc_prefetch_depth": 2,
773
+ "logits_load_kd_loss_alpha": 1.0,
774
+ "logits_load_ignore_errors": false,
775
+ "log_straggler": false,
776
+ "straggler_ctrlr_port": 65535,
777
+ "straggler_minmax_count": 1,
778
+ "disable_straggler_on_startup": false,
779
+ "run_workload_inspector_server": false,
780
+ "inference_batch_times_seqlen_threshold": -1,
781
+ "max_tokens_to_oom": 12000,
782
+ "output_bert_embeddings": false,
783
+ "bert_embedder_type": "megatron",
784
+ "cuda_graph_modules": [],
785
+ "use_legacy_static_engine": false,
786
+ "inference_max_requests": 8,
787
+ "inference_max_seq_length": 2560,
788
+ "inference_dynamic_batching": false,
789
+ "inference_dynamic_batching_buffer_size_gb": 40.0,
790
+ "inference_dynamic_batching_paused_buffer_size_gb": null,
791
+ "inference_dynamic_batching_mamba_memory_ratio": null,
792
+ "inference_dynamic_batching_block_size": 256,
793
+ "inference_dynamic_batching_max_requests": null,
794
+ "inference_dynamic_batching_max_tokens": null,
795
+ "inference_dynamic_batching_num_cuda_graphs": 16,
796
+ "inference_dynamic_batching_track_paused_request_events": false,
797
+ "inference_dynamic_batching_track_generated_token_events": false,
798
+ "decode_only_cuda_graphs": false,
799
+ "inference_cuda_graph_all_prefills": false,
800
+ "inference_cuda_graph_max_tokens": 512,
801
+ "inference_dynamic_batching_unified_memory_level": 0,
802
+ "enable_chunked_prefill": false,
803
+ "num_speculative_tokens": 0,
804
+ "inference_dynamic_batching_enable_prefix_caching": false,
805
+ "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
806
+ "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
807
+ "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
808
+ "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
809
+ "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
810
+ "inference_dynamic_batching_media_cache_routing_weight": 1.0,
811
+ "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
812
+ "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
813
+ "inference_dynamic_batching_prefix_caching_mamba_gb": null,
814
+ "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
815
+ "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
816
+ "inference_dynamic_batching_sampling_backend": "torch",
817
+ "offset_sampling_seed_by_dp_rank": true,
818
+ "inference_dynamic_batching_async_sched_mode": "async",
819
+ "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
820
+ "inference_logging_step_interval": 0,
821
+ "inference_text_gen_server_logging": false,
822
+ "inference_wandb_logging": false,
823
+ "inference_coordinator_port": null,
824
+ "mamba_inference_conv_states_dtype": {
825
+ "__torch_dtype__": "bfloat16"
826
+ },
827
+ "mamba_inference_ssm_states_dtype": {
828
+ "__torch_dtype__": "bfloat16"
829
+ },
830
+ "inference_use_synchronous_zmq_collectives": false,
831
+ "inference_disable_ep_consensus": false,
832
+ "inference_shards": null,
833
+ "fp4_param_gather": false,
834
+ "fp8_param_gather": false,
835
+ "te_precision_config_file": null,
836
+ "enable_experimental": false,
837
+ "spec": null,
838
+ "hybrid_layer_pattern": null,
839
+ "hybrid_override_pattern": null,
840
+ "yaml_cfg": null,
841
+ "use_precision_aware_optimizer": false,
842
+ "main_grads_dtype": {
843
+ "__torch_dtype__": "float32"
844
+ },
845
+ "main_params_dtype": {
846
+ "__torch_dtype__": "float32"
847
+ },
848
+ "exp_avg_dtype": {
849
+ "__torch_dtype__": "float32"
850
+ },
851
+ "exp_avg_sq_dtype": {
852
+ "__torch_dtype__": "float32"
853
+ },
854
+ "megatron_fsdp_main_params_dtype": {
855
+ "__torch_dtype__": "float32"
856
+ },
857
+ "megatron_fsdp_main_grads_dtype": null,
858
+ "megatron_fsdp_grad_comm_dtype": null,
859
+ "megatron_fsdp_enable_fine_grained_param_gather": false,
860
+ "megatron_fsdp_max_pool_double_buffer": false,
861
+ "fsdp_db_use_persist_buf_on_alloc_fail": false,
862
+ "enable_one_logger": true,
863
+ "one_logger_project": "megatron-lm",
864
+ "one_logger_run_name": null,
865
+ "one_logger_async": false,
866
+ "app_tag_run_name": null,
867
+ "app_tag_run_version": "0.0.0",
868
+ "inprocess_restart": false,
869
+ "inprocess_max_iterations": null,
870
+ "inprocess_monitor_thread_interval": 1.0,
871
+ "inprocess_monitor_process_interval": 1.0,
872
+ "inprocess_progress_watchdog_interval": 1.0,
873
+ "inprocess_heartbeat_interval": 30,
874
+ "inprocess_soft_timeout": 60,
875
+ "inprocess_hard_timeout": 90,
876
+ "inprocess_heartbeat_timeout": 60,
877
+ "inprocess_barrier_timeout": 120,
878
+ "inprocess_completion_timeout": 120,
879
+ "inprocess_last_call_wait": 1,
880
+ "inprocess_termination_grace_time": 1,
881
+ "inprocess_granularity": "node",
882
+ "inprocess_active_world_size": 16,
883
+ "inprocess_empty_cuda_cache": false,
884
+ "enable_ft_package": false,
885
+ "calc_ft_timeouts": false,
886
+ "ft_num_warmup_iters": 5,
887
+ "error_injection_rate": 0,
888
+ "error_injection_type": "transient_error",
889
+ "rerun_mode": "validate_results",
890
+ "check_for_spiky_loss": false,
891
+ "enable_msc": false,
892
+ "disable_msc_deprecated": false,
893
+ "sft": false,
894
+ "sft_tokenizer_prompt_format": "nemotron-h-aligned",
895
+ "sft_mock_dataset_config_json": null,
896
+ "use_varlen_dataset": false,
897
+ "varlen_sbhd_validation": false,
898
+ "varlen_mock_dataset_config_json": null,
899
+ "fault_injector_ranks": null,
900
+ "fault_injector_num_ranks": null,
901
+ "fault_injector_fault_types": null,
902
+ "fault_injector_fault_probabilities": null,
903
+ "fault_injector_fault_delay": null,
904
+ "fault_injector_delay_start_iteration": null,
905
+ "fault_injector_mtti_seconds": null,
906
+ "fault_injector_offset_seconds": null,
907
+ "fault_injector_seed": null,
908
+ "architecture": "full_bandwidth",
909
+ "architecture_module": [],
910
+ "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
911
+ "training_recipe": "paper2604_21106",
912
+ "recipe_overrides": {},
913
+ "recipe_requested_batch_tokens": 262144,
914
+ "paper_reference_width": 640,
915
+ "paper_reference_tokens": 983315200,
916
+ "paper_reference_batch_tokens": 262144,
917
+ "paper_embedding_lr": 0.3,
918
+ "paper_unembedding_lr": 0.004,
919
+ "paper_norm_lr": 0.005,
920
+ "paper_other_lr": 0.005,
921
+ "online_data": true,
922
+ "online_pack_sequences": false,
923
+ "online_dataset": "HuggingFaceFW/fineweb-edu",
924
+ "online_dataset_subset": "sample-100BT",
925
+ "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
926
+ "online_source_split": "train",
927
+ "online_text_column": "text",
928
+ "online_validation_fraction": 0.001,
929
+ "online_split_seed": 260421106,
930
+ "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
931
+ "online_tokenizer_batch_size": 256,
932
+ "online_shuffle_buffer_records": 128,
933
+ "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
934
+ "online_prefetch_batches": 16,
935
+ "online_source_mode": "local",
936
+ "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
937
+ "online_validation_manifest": null,
938
+ "online_latency_log_dir": null,
939
+ "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
940
+ "eval_at_start": true,
941
+ "final_eval_iters": 270,
942
+ "backbone_profile": "paper2604_vanilla",
943
+ "aux_mtp_horizon": 1,
944
+ "aux_mtp_rank": 128,
945
+ "aux_mtp_loss_weight": 0.3,
946
+ "feedback_loss_weight": 1.0,
947
+ "feedback_layers": null,
948
+ "feedback_jitter_sigma": 0.0,
949
+ "feedback_prefix_mixin": false,
950
+ "feedback_pass_mixture": "3:1.0",
951
+ "feedback_progressive_stages": "[]",
952
+ "feedback_eval_passes": 3,
953
+ "feedback_z_loss_coefficient": 0.0,
954
+ "feedback_z_loss_start_fraction": 1.0,
955
+ "feedback_log_hidden_change": false,
956
+ "feedback_scalar_lr": 0.0005,
957
+ "feedback_scalar_weight_decay": 0.0,
958
+ "feedback_normuon_beta2": 0.95,
959
+ "adaptive_muon_moment2_method": "normuon",
960
+ "adaptive_muon_beta2": 0.95,
961
+ "adaptive_muon_eps": 1e-10,
962
+ "_is_global_batch_size_explicitly_specified": true,
963
+ "rank": 0,
964
+ "world_size": 16,
965
+ "use_dist_ckpt": false,
966
+ "gtp_weight_remat_size": 1,
967
+ "transformer_pipeline_model_parallel_size": 1,
968
+ "data_parallel_size": 16,
969
+ "virtual_pipeline_model_parallel_size": null,
970
+ "params_dtype": {
971
+ "__torch_dtype__": "bfloat16"
972
+ },
973
+ "consumed_train_samples": 1536000,
974
+ "skipped_train_samples": 0,
975
+ "consumed_valid_samples": 30976,
976
+ "variable_seq_lengths": false,
977
+ "expert_gtp_weight_remat_size": 1,
978
+ "use_layer_wise_distributed_optimizer": false,
979
+ "recipe_optimizer": {
980
+ "name": "muonh",
981
+ "matrix_lr": 0.014,
982
+ "scalar_lr": null,
983
+ "matrix_weight_decay": 0.0,
984
+ "scalar_weight_decay": 0.0,
985
+ "embedding_lr": 0.3,
986
+ "unembedding_lr": 0.004,
987
+ "norm_lr": 0.005,
988
+ "other_lr": 0.005,
989
+ "betas": [
990
+ 0.8,
991
+ 0.95
992
+ ],
993
+ "eps": 1e-10,
994
+ "normuon_momentum": 0.95,
995
+ "normuon_beta2": 0.95,
996
+ "normuon_nesterov": true,
997
+ "normuon_adjust_lr": "spectral_norm",
998
+ "max_grad_norm": null
999
+ },
1000
+ "recipe_schedule": {
1001
+ "kind": "linear",
1002
+ "warmup_steps": 0,
1003
+ "cooldown_fraction": 0.0,
1004
+ "cooldown_style": "linear",
1005
+ "final_lr_fraction": 0.1,
1006
+ "z_loss_coefficient": 0.0,
1007
+ "adamc_weight_decay": false
1008
+ },
1009
+ "paper_base_matrix_lr": 0.014,
1010
+ "paper_transfer": {
1011
+ "__dataclass__": [
1012
+ "loop_transformer_backends.recipes",
1013
+ "Paper2604Transfer",
1014
+ {
1015
+ "width": 1408,
1016
+ "total_tokens": 7000031232,
1017
+ "batch_tokens": 262144,
1018
+ "reference_width": 640,
1019
+ "reference_tokens": 983315200,
1020
+ "reference_batch_tokens": 262144
1021
+ }
1022
+ ]
1023
+ },
1024
+ "training_recipe_metadata": {
1025
+ "recipe": "paper2604_21106",
1026
+ "width": 1408,
1027
+ "total_tokens": 7000031232,
1028
+ "batch_tokens": 262144,
1029
+ "reference_width": 640,
1030
+ "reference_tokens": 983315200,
1031
+ "reference_batch_tokens": 262144,
1032
+ "base_lrs": {
1033
+ "matrix": 0.014,
1034
+ "embedding": 0.3,
1035
+ "unembedding": 0.004,
1036
+ "norm": 0.005,
1037
+ "other": 0.005
1038
+ },
1039
+ "peak_lrs": {
1040
+ "matrix": 0.007470638533717786,
1041
+ "embedding": 0.2022599587389726,
1042
+ "unembedding": 0.0026967994498529684,
1043
+ "norm": 0.00337099931231621,
1044
+ "other": 0.00337099931231621
1045
+ },
1046
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
1047
+ "tied_embedding_policy": "unembedding (one AdamW update)",
1048
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
1049
+ "weight_decay": 0.0,
1050
+ "warmup_steps": 0,
1051
+ "final_lr_fraction": 0.1,
1052
+ "optimizer": {
1053
+ "name": "muonh",
1054
+ "matrix_lr": 0.014,
1055
+ "scalar_lr": null,
1056
+ "matrix_weight_decay": 0.0,
1057
+ "scalar_weight_decay": 0.0,
1058
+ "embedding_lr": 0.3,
1059
+ "unembedding_lr": 0.004,
1060
+ "norm_lr": 0.005,
1061
+ "other_lr": 0.005,
1062
+ "betas": [
1063
+ 0.8,
1064
+ 0.95
1065
+ ],
1066
+ "eps": 1e-10,
1067
+ "normuon_momentum": 0.95,
1068
+ "normuon_beta2": 0.95,
1069
+ "normuon_nesterov": true,
1070
+ "normuon_adjust_lr": "spectral_norm",
1071
+ "max_grad_norm": null
1072
+ },
1073
+ "schedule": {
1074
+ "kind": "linear",
1075
+ "warmup_steps": 0,
1076
+ "cooldown_fraction": 0.0,
1077
+ "cooldown_style": "linear",
1078
+ "final_lr_fraction": 0.1,
1079
+ "z_loss_coefficient": 0.0,
1080
+ "adamc_weight_decay": false
1081
+ },
1082
+ "group_weight_decay": {
1083
+ "matrix": 0.0,
1084
+ "embedding": 0.0,
1085
+ "unembedding": 0.0,
1086
+ "norm": 0.0,
1087
+ "other": 0.0
1088
+ },
1089
+ "recipe_overrides": {},
1090
+ "requested_batch_tokens": 262144,
1091
+ "sequence_length": 2048,
1092
+ "batch_assumptions": {
1093
+ "global_batch_tokens": 262144,
1094
+ "sequence_length": 2048,
1095
+ "round_up": false
1096
+ },
1097
+ "lr_transfer": "mup_hyperp_released_source",
1098
+ "assumptions": [],
1099
+ "backend_kernel_assumptions": {},
1100
+ "paper_settings_overridden": false,
1101
+ "model": {
1102
+ "backbone_profile": "paper2604_vanilla",
1103
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
1104
+ "width_scale": 22,
1105
+ "hidden_size": 1408,
1106
+ "num_layers": 20,
1107
+ "head_dim": 128,
1108
+ "num_attention_heads": 11,
1109
+ "num_kv_heads": 11,
1110
+ "ffn_hidden_size": 5632,
1111
+ "activation": "squared_relu",
1112
+ "attention": "full_causal",
1113
+ "attention_output_gate": false,
1114
+ "qk_norm": "functional_rmsnorm_after_rope",
1115
+ "norm_epsilon": 1e-06,
1116
+ "model_level_norms": [
1117
+ "embedding",
1118
+ "recur",
1119
+ "final"
1120
+ ],
1121
+ "rope_base": 10000.0,
1122
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
1123
+ "bias": false,
1124
+ "dropout": 0.0,
1125
+ "residual_multiplier": 1.0,
1126
+ "residual_and_norm_scale_dtype": "float32",
1127
+ "projection_dtype": "bfloat16 (production)",
1128
+ "tied_embeddings": false,
1129
+ "vocab_size": 32001,
1130
+ "padded_vocab_size": 32064,
1131
+ "logit_softcap": 15.0,
1132
+ "crop_padding_before_loss": true,
1133
+ "initialization": {
1134
+ "embedding_normal_std": 1.0,
1135
+ "head_normal_std": 0.001,
1136
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
1137
+ "norm_scale": 1.0
1138
+ },
1139
+ "parameter_counts": {
1140
+ "non_embedding_matrix_formula": 475791360,
1141
+ "non_embedding_paper_N": 475851904,
1142
+ "total_trainable": 570111872,
1143
+ "feedback_parameters": 3967744
1144
+ },
1145
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
1146
+ "execution_architecture": "full_bandwidth",
1147
+ "objective": "latent_feedback_causal_lm",
1148
+ "feedback_enabled": true,
1149
+ "train_passes": 3,
1150
+ "eval_passes": 3,
1151
+ "feedback_pass_mixture": {
1152
+ "3": 1.0
1153
+ },
1154
+ "feedback_progressive_stages": []
1155
+ }
1156
+ },
1157
+ "backbone_metadata": {
1158
+ "backbone_profile": "paper2604_vanilla",
1159
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
1160
+ "width_scale": 22,
1161
+ "hidden_size": 1408,
1162
+ "num_layers": 20,
1163
+ "head_dim": 128,
1164
+ "num_attention_heads": 11,
1165
+ "num_kv_heads": 11,
1166
+ "ffn_hidden_size": 5632,
1167
+ "activation": "squared_relu",
1168
+ "attention": "full_causal",
1169
+ "attention_output_gate": false,
1170
+ "qk_norm": "functional_rmsnorm_after_rope",
1171
+ "norm_epsilon": 1e-06,
1172
+ "model_level_norms": [
1173
+ "embedding",
1174
+ "recur",
1175
+ "final"
1176
+ ],
1177
+ "rope_base": 10000.0,
1178
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
1179
+ "bias": false,
1180
+ "dropout": 0.0,
1181
+ "residual_multiplier": 1.0,
1182
+ "residual_and_norm_scale_dtype": "float32",
1183
+ "projection_dtype": "bfloat16 (production)",
1184
+ "tied_embeddings": false,
1185
+ "vocab_size": 32001,
1186
+ "padded_vocab_size": 32064,
1187
+ "logit_softcap": 15.0,
1188
+ "crop_padding_before_loss": true,
1189
+ "initialization": {
1190
+ "embedding_normal_std": 1.0,
1191
+ "head_normal_std": 0.001,
1192
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
1193
+ "norm_scale": 1.0
1194
+ },
1195
+ "parameter_counts": {
1196
+ "non_embedding_matrix_formula": 475791360,
1197
+ "non_embedding_paper_N": 475851904,
1198
+ "total_trainable": 570111872,
1199
+ "feedback_parameters": 3967744
1200
+ },
1201
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
1202
+ "execution_architecture": "full_bandwidth",
1203
+ "objective": "latent_feedback_causal_lm",
1204
+ "feedback_enabled": true,
1205
+ "train_passes": 3,
1206
+ "eval_passes": 3,
1207
+ "feedback_pass_mixture": {
1208
+ "3": 1.0
1209
+ },
1210
+ "feedback_progressive_stages": []
1211
+ },
1212
+ "evaluation_metadata": {
1213
+ "eval_at_start": true,
1214
+ "initial_iters": 2,
1215
+ "intermediate_iters": 2,
1216
+ "final_validation_iters": 270,
1217
+ "test_iters": 0,
1218
+ "global_batch_size": 128,
1219
+ "sequence_length": 2048,
1220
+ "intermediate_input_tokens": 524288,
1221
+ "final_validation_sequences": 34560,
1222
+ "final_validation_input_tokens": 70778880,
1223
+ "validation_iterator": "continues_existing_held_out_iterator"
1224
+ },
1225
+ "iteration": 0,
1226
+ "num_floating_point_operations_so_far": 0,
1227
+ "do_train": 1,
1228
+ "do_valid": 1,
1229
+ "do_test": 0,
1230
+ "curr_iteration": 11999
1231
+ },
1232
+ "non_tensor_state": {
1233
+ "decoder.layers.0.self_attention.linear_proj._extra_state": null,
1234
+ "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
1235
+ "decoder.layers.0.mlp.linear_fc1._extra_state": null,
1236
+ "decoder.layers.0.mlp.linear_fc2._extra_state": null,
1237
+ "decoder.layers.1.self_attention.linear_proj._extra_state": null,
1238
+ "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
1239
+ "decoder.layers.1.mlp.linear_fc1._extra_state": null,
1240
+ "decoder.layers.1.mlp.linear_fc2._extra_state": null,
1241
+ "decoder.layers.2.self_attention.linear_proj._extra_state": null,
1242
+ "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
1243
+ "decoder.layers.2.mlp.linear_fc1._extra_state": null,
1244
+ "decoder.layers.2.mlp.linear_fc2._extra_state": null,
1245
+ "decoder.layers.3.self_attention.linear_proj._extra_state": null,
1246
+ "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
1247
+ "decoder.layers.3.mlp.linear_fc1._extra_state": null,
1248
+ "decoder.layers.3.mlp.linear_fc2._extra_state": null,
1249
+ "decoder.layers.4.self_attention.linear_proj._extra_state": null,
1250
+ "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
1251
+ "decoder.layers.4.mlp.linear_fc1._extra_state": null,
1252
+ "decoder.layers.4.mlp.linear_fc2._extra_state": null,
1253
+ "decoder.layers.5.self_attention.linear_proj._extra_state": null,
1254
+ "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
1255
+ "decoder.layers.5.mlp.linear_fc1._extra_state": null,
1256
+ "decoder.layers.5.mlp.linear_fc2._extra_state": null,
1257
+ "decoder.layers.6.self_attention.linear_proj._extra_state": null,
1258
+ "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
1259
+ "decoder.layers.6.mlp.linear_fc1._extra_state": null,
1260
+ "decoder.layers.6.mlp.linear_fc2._extra_state": null,
1261
+ "decoder.layers.7.self_attention.linear_proj._extra_state": null,
1262
+ "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
1263
+ "decoder.layers.7.mlp.linear_fc1._extra_state": null,
1264
+ "decoder.layers.7.mlp.linear_fc2._extra_state": null,
1265
+ "decoder.layers.8.self_attention.linear_proj._extra_state": null,
1266
+ "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
1267
+ "decoder.layers.8.mlp.linear_fc1._extra_state": null,
1268
+ "decoder.layers.8.mlp.linear_fc2._extra_state": null,
1269
+ "decoder.layers.9.self_attention.linear_proj._extra_state": null,
1270
+ "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
1271
+ "decoder.layers.9.mlp.linear_fc1._extra_state": null,
1272
+ "decoder.layers.9.mlp.linear_fc2._extra_state": null,
1273
+ "decoder.layers.10.self_attention.linear_proj._extra_state": null,
1274
+ "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
1275
+ "decoder.layers.10.mlp.linear_fc1._extra_state": null,
1276
+ "decoder.layers.10.mlp.linear_fc2._extra_state": null,
1277
+ "decoder.layers.11.self_attention.linear_proj._extra_state": null,
1278
+ "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
1279
+ "decoder.layers.11.mlp.linear_fc1._extra_state": null,
1280
+ "decoder.layers.11.mlp.linear_fc2._extra_state": null,
1281
+ "decoder.layers.12.self_attention.linear_proj._extra_state": null,
1282
+ "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
1283
+ "decoder.layers.12.mlp.linear_fc1._extra_state": null,
1284
+ "decoder.layers.12.mlp.linear_fc2._extra_state": null,
1285
+ "decoder.layers.13.self_attention.linear_proj._extra_state": null,
1286
+ "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
1287
+ "decoder.layers.13.mlp.linear_fc1._extra_state": null,
1288
+ "decoder.layers.13.mlp.linear_fc2._extra_state": null,
1289
+ "decoder.layers.14.self_attention.linear_proj._extra_state": null,
1290
+ "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
1291
+ "decoder.layers.14.mlp.linear_fc1._extra_state": null,
1292
+ "decoder.layers.14.mlp.linear_fc2._extra_state": null,
1293
+ "decoder.layers.15.self_attention.linear_proj._extra_state": null,
1294
+ "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
1295
+ "decoder.layers.15.mlp.linear_fc1._extra_state": null,
1296
+ "decoder.layers.15.mlp.linear_fc2._extra_state": null,
1297
+ "decoder.layers.16.self_attention.linear_proj._extra_state": null,
1298
+ "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
1299
+ "decoder.layers.16.mlp.linear_fc1._extra_state": null,
1300
+ "decoder.layers.16.mlp.linear_fc2._extra_state": null,
1301
+ "decoder.layers.17.self_attention.linear_proj._extra_state": null,
1302
+ "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
1303
+ "decoder.layers.17.mlp.linear_fc1._extra_state": null,
1304
+ "decoder.layers.17.mlp.linear_fc2._extra_state": null,
1305
+ "decoder.layers.18.self_attention.linear_proj._extra_state": null,
1306
+ "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
1307
+ "decoder.layers.18.mlp.linear_fc1._extra_state": null,
1308
+ "decoder.layers.18.mlp.linear_fc2._extra_state": null,
1309
+ "decoder.layers.19.self_attention.linear_proj._extra_state": null,
1310
+ "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
1311
+ "decoder.layers.19.mlp.linear_fc1._extra_state": null,
1312
+ "decoder.layers.19.mlp.linear_fc2._extra_state": null,
1313
+ "output_layer._extra_state": null,
1314
+ "feedback.state_proj._extra_state": null,
1315
+ "feedback.token_gate_proj._extra_state": null
1316
+ },
1317
+ "tensor_parallel_size": 1,
1318
+ "pipeline_parallel_size": 1,
1319
+ "training_tokens": 3145728000,
1320
+ "training_budget_tokens": 7000031232,
1321
+ "effective_launch_overrides": {
1322
+ "NNODES": 2,
1323
+ "MICRO_BATCH_SIZE": 8,
1324
+ "GLOBAL_BATCH_SIZE": 128,
1325
+ "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
1326
+ "SAVE_RETAIN_INTERVAL": 4000
1327
+ },
1328
+ "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
1329
+ "tokenizer": {
1330
+ "repository": "NousResearch/Llama-2-7b-hf",
1331
+ "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
1332
+ "added_tokens": [
1333
+ "<|assistant_end|>"
1334
+ ],
1335
+ "active_vocab_size": 32001,
1336
+ "padded_vocab_size": 32064
1337
+ },
1338
+ "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
1339
+ }
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/launch_config.env ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
2
+ source "$(dirname "${BASH_SOURCE[0]}")/common.env"
3
+ RUN_NAME=r4_parent_fbw_k3_20l_7b
4
+ RESUME_POLICY=fresh
5
+ REQUIRE_CHECKPOINT=false
6
+ WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/manifest.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "files": {
3
+ "README.md": {
4
+ "size": 3184,
5
+ "sha256": "68ff162efc992c025c73926b107c39fc8986fa49d5197c6ab1b7d5885992ab4d"
6
+ },
7
+ "config.json": {
8
+ "size": 50495,
9
+ "sha256": "ce8c677682f276b7b64a0b70809ecec0a6420268abb665e53f1f1fa07c9346ab"
10
+ },
11
+ "launch_config.env": {
12
+ "size": 270,
13
+ "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
14
+ },
15
+ "model.safetensors": {
16
+ "size": 1140359624,
17
+ "sha256": "831348a36bcb251f8b896efa57cc06ca6a696a55fbc041485b37d396c62daf9d"
18
+ },
19
+ "training_recipe.json": {
20
+ "size": 3782,
21
+ "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
22
+ }
23
+ },
24
+ "note": "Manifest does not hash itself."
25
+ }
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:831348a36bcb251f8b896efa57cc06ca6a696a55fbc041485b37d396c62daf9d
3
+ size 1140359624
FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/training_recipe.json ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recipe": "paper2604_21106",
3
+ "width": 1408,
4
+ "total_tokens": 7000031232,
5
+ "batch_tokens": 262144,
6
+ "reference_width": 640,
7
+ "reference_tokens": 983315200,
8
+ "reference_batch_tokens": 262144,
9
+ "base_lrs": {
10
+ "matrix": 0.014,
11
+ "embedding": 0.3,
12
+ "unembedding": 0.004,
13
+ "norm": 0.005,
14
+ "other": 0.005
15
+ },
16
+ "peak_lrs": {
17
+ "matrix": 0.007470638533717786,
18
+ "embedding": 0.2022599587389726,
19
+ "unembedding": 0.0026967994498529684,
20
+ "norm": 0.00337099931231621,
21
+ "other": 0.00337099931231621
22
+ },
23
+ "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
24
+ "tied_embedding_policy": "unembedding (one AdamW update)",
25
+ "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
26
+ "weight_decay": 0.0,
27
+ "warmup_steps": 0,
28
+ "final_lr_fraction": 0.1,
29
+ "optimizer": {
30
+ "name": "muonh",
31
+ "matrix_lr": 0.014,
32
+ "scalar_lr": null,
33
+ "matrix_weight_decay": 0.0,
34
+ "scalar_weight_decay": 0.0,
35
+ "embedding_lr": 0.3,
36
+ "unembedding_lr": 0.004,
37
+ "norm_lr": 0.005,
38
+ "other_lr": 0.005,
39
+ "betas": [
40
+ 0.8,
41
+ 0.95
42
+ ],
43
+ "eps": 1e-10,
44
+ "normuon_momentum": 0.95,
45
+ "normuon_beta2": 0.95,
46
+ "normuon_nesterov": true,
47
+ "normuon_adjust_lr": "spectral_norm",
48
+ "max_grad_norm": null
49
+ },
50
+ "schedule": {
51
+ "kind": "linear",
52
+ "warmup_steps": 0,
53
+ "cooldown_fraction": 0.0,
54
+ "cooldown_style": "linear",
55
+ "final_lr_fraction": 0.1,
56
+ "z_loss_coefficient": 0.0,
57
+ "adamc_weight_decay": false
58
+ },
59
+ "group_weight_decay": {
60
+ "matrix": 0.0,
61
+ "embedding": 0.0,
62
+ "unembedding": 0.0,
63
+ "norm": 0.0,
64
+ "other": 0.0
65
+ },
66
+ "recipe_overrides": {},
67
+ "requested_batch_tokens": 262144,
68
+ "sequence_length": 2048,
69
+ "batch_assumptions": {
70
+ "global_batch_tokens": 262144,
71
+ "sequence_length": 2048,
72
+ "round_up": false
73
+ },
74
+ "lr_transfer": "mup_hyperp_released_source",
75
+ "assumptions": [],
76
+ "backend_kernel_assumptions": {},
77
+ "paper_settings_overridden": false,
78
+ "model": {
79
+ "backbone_profile": "paper2604_vanilla",
80
+ "source_commit": "122366472a31e57982a2877eeee10bb115477713",
81
+ "width_scale": 22,
82
+ "hidden_size": 1408,
83
+ "num_layers": 20,
84
+ "head_dim": 128,
85
+ "num_attention_heads": 11,
86
+ "num_kv_heads": 11,
87
+ "ffn_hidden_size": 5632,
88
+ "activation": "squared_relu",
89
+ "attention": "full_causal",
90
+ "attention_output_gate": false,
91
+ "qk_norm": "functional_rmsnorm_after_rope",
92
+ "norm_epsilon": 1e-06,
93
+ "model_level_norms": [
94
+ "embedding",
95
+ "recur",
96
+ "final"
97
+ ],
98
+ "rope_base": 10000.0,
99
+ "rope_convention": "split_half_negative_angle_bf16_cos_sin",
100
+ "bias": false,
101
+ "dropout": 0.0,
102
+ "residual_multiplier": 1.0,
103
+ "residual_and_norm_scale_dtype": "float32",
104
+ "projection_dtype": "bfloat16 (production)",
105
+ "tied_embeddings": false,
106
+ "vocab_size": 32001,
107
+ "padded_vocab_size": 32064,
108
+ "logit_softcap": 15.0,
109
+ "crop_padding_before_loss": true,
110
+ "initialization": {
111
+ "embedding_normal_std": 1.0,
112
+ "head_normal_std": 0.001,
113
+ "matrix_uniform_bound": "sqrt(3/fan_in)",
114
+ "norm_scale": 1.0
115
+ },
116
+ "parameter_counts": {
117
+ "non_embedding_matrix_formula": 475791360,
118
+ "non_embedding_paper_N": 475851904,
119
+ "total_trainable": 570111872,
120
+ "feedback_parameters": 3967744
121
+ },
122
+ "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
123
+ "execution_architecture": "full_bandwidth",
124
+ "objective": "latent_feedback_causal_lm",
125
+ "feedback_enabled": true,
126
+ "train_passes": 3,
127
+ "eval_passes": 3,
128
+ "feedback_pass_mixture": {
129
+ "3": 1.0
130
+ },
131
+ "feedback_progressive_stages": []
132
+ }
133
+ }