diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/NAMING.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/NAMING.json
new file mode 100644
index 0000000000000000000000000000000000000000..95614088facc98daca132850564b34167a8c11c5
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/NAMING.json
@@ -0,0 +1,10 @@
+{
+  "wandb_id": "ljsfjz37",
+  "previous_wandb_name": "fbw-layers2-11-20-k3-s42-7b--ljsfjz37",
+  "training_run_name": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "canonical_name": "FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37",
+  "architecture": "full_bandwidth",
+  "hf_old_paths": [
+    "fbw_L2L11L20_7b_seed42_20260920-023045"
+  ]
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..947dfe5eb8111bff54d08fa50c58c450dc4c844a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 12000
+
+Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_12000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d723764760626c404208ae2ed8027590744d5de
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 12000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 1536000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 12000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 3145728000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..1ebeed0a75035fb5d286813e5f23f9660f1d3da1
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 12000,
+  "files": {
+    "README.md": {
+      "sha256": "ecfcad9dcd20100fe9c87281508367495a2e47f6ec745b20fad8455b5879fd1f",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "6e6a9ced87ea8f55c9ab0533b901e84e97991d5aa0838f0b29ee213e9639793a",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_12000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2a0c0e6b9cd47f0d223e6a810a4de65eea332585
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cf078511cf90a669c48751691728f5a4c7537a4db065cc8f75c82850ccdef00
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_12000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..273b561c8ac370aca268acc1fe3472ccc7b54de2
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 16000
+
+Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_16000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..73442569517901d6c1e2b609d29d6e0ef5a43ea2
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 16000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 2048000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 16000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 4194304000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..c8b3b0b13414647c731b487a1b5ce8c5fbe9a791
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 16000,
+  "files": {
+    "README.md": {
+      "sha256": "d3f4a9de16bf052561064000ad77015e88319f11afa1b6c0d01d3628ec9dc4b6",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "508f1517705df16a8bd79f1b0c9ed5c610679485964d5c670e95fbafa6f83d9a",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_16000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bcc1d191aa1b536e961334695bc12cdad81ec371
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3131e4b396f257888a76b4ca4f2c2c5c5540317dacee0ef9f05574246ea8639
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_16000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b12d0c46ee535d9fe86aba70b6c977149c585882
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 20000
+
+Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_20000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf723394e3f7b2cb6e7bfca990e5616c082ffb50
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 20000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 2560000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 20000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 5242880000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..66fbc2a2cec27d70dd6ea9f74f13c4031a923c39
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 20000,
+  "files": {
+    "README.md": {
+      "sha256": "e5c48096d83b5fbb9c8ba691b37f3f483a5cf1c67ef37d9dd0bc90c072fe1b18",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "0b26f395c2623ab007efcafec34a04fd0b3238d47d690b1bc2cf2cb9d2116737",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_20000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..19e255644632f850c2d06c7f762a883df2bfb2a3
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d635c166069ce58adfbbedac40944c55181ef024e026a0f837d5871b40a38a
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_20000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1a00642e59c12505891c2401695c0598e57e9666
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 24000
+
+Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_24000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f1f620bb606ce0af10cbf4c59997784905acf1f
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 24000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 3072000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 24000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 6291456000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..e06ad2f348ee0f36c2d565e41c0070765d578204
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 24000,
+  "files": {
+    "README.md": {
+      "sha256": "a2d3faf19e2eea6312eb85940944b0afbe84dfcc25d302616f2c66549bfe8ad2",
+      "size": 2699
+    },
+    "config.json": {
+      "sha256": "5619a51676440252ea6ff2c78de18c37322142cacc04d0188457faa0e28c4d8d",
+      "size": 12273
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_24000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..87b58bfbe9db0c9d59bbd65642bc6e1563e6dd6a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67022a6890c1607aaea42a052990602c446b89a665540c79db73ef052543f623
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_24000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..746817cd8c25d22c14a693da397481709df964c2
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 26703
+
+Final checkpoint at step **26,703**, after **7,000,031,232 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_26703")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..e65bc6da2e927ef82209e005fc62c2b9756e90de
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 26703,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 3417984,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": true,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 26703,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 7000031232,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..547076128536c164949e2f3a8f895bb6b328a540
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 26703,
+  "files": {
+    "README.md": {
+      "sha256": "d5a7044737b80db05eaf39029237d2df47cfd6135677146decb1997732ae5266",
+      "size": 2692
+    },
+    "config.json": {
+      "sha256": "0be3be4a6e3da7092a2bc29821ced20389b0298ede9879790c9762c65ff47ea3",
+      "size": 12272
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_26703",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..04eb7323b41f24ccc12e59fdb40f3654bfa0b47c
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4457478d9c6ef9c5ef7e69fa26c007130a1b4f302ece02e19ba92bbcecdb1041
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_26703/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1c15ec7ccfdae2f184046ee38e505b769cb214ed
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 4000
+
+Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_4000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..87e48e617ced0e7b1ca3dae16ba7e64cd430bbe0
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 4000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 512000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 4000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 1048576000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..8267f55c73dbacee44103fc4a1cda5a145bfe9d3
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 4000,
+  "files": {
+    "README.md": {
+      "sha256": "754c86bf86065e0f121d51b1f3046858ead7aaea160a46caa6db1ddc72cb127d",
+      "size": 2696
+    },
+    "config.json": {
+      "sha256": "880053416c618d5a37cb0de24ba8dfdd88418832d970f89d2e55311474136a81",
+      "size": 12270
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_4000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f2507854e4f85ec75aec4167d84f19a85b820c79
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65fbfd4501dfc4d03939ceecb26ad41ff10050b40e602e58c56c0b734afd9200
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_4000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/README.md b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d910fc9cefc8cec598972ab154830f66a4dfc988
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/README.md
@@ -0,0 +1,35 @@
+# fbw_L2L11L20_7b_seed42_20260920-023045 — step 8000
+
+Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **576,059,264 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget.
+
+Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained.
+
+Training revision: `9f3c22f21c6026055ba69a39af5f285f09d3cfc8`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8) with `src` on `PYTHONPATH` and its dependencies installed.
+
+The existing project native feedback inference adapter preserves the saved feedback layer selection and fusion projection. It maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. Use native_loading_args exactly as recorded.
+
+## Loading
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+export_dir = Path("/path/to/step_8000")
+config = json.loads((export_dir / "config.json").read_text())
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+from types import SimpleNamespace
+from loop_transformer_backends.benchmarks.models import build_native
+model, metadata = build_native({
+    "args": SimpleNamespace(**config["native_loading_args"]),
+    "model": state | config["non_tensor_state_dict_entries"],
+    "iteration": config["checkpoint_iteration"],
+})  # Existing project inference mapping, strict=True, assign=True internally.
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading.
+
+No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed.
+
+W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/ljsfjz37). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included.
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/config.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..09b460ef5bceccf546d2bfb037e36dee613d0a86
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/config.json
@@ -0,0 +1,289 @@
+{
+  "activation": "squared_relu",
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 8000,
+  "checkpoint_version": 3.0,
+  "consumed_train_samples": 1024000,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "effective_overrides": {
+    "EXIT_INTERVAL": 26703,
+    "FEEDBACK_EVAL_PASSES": 3,
+    "FEEDBACK_JITTER_SIGMA": 0.0,
+    "FEEDBACK_LAYERS": "2,11,20",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": false,
+    "FEEDBACK_LOSS_WEIGHT": 1.0,
+    "FEEDBACK_NORMUON_BETA2": 0.95,
+    "FEEDBACK_PASS_MIXTURE": "3:1.0",
+    "FEEDBACK_PREFIX_MIXIN": false,
+    "FEEDBACK_PROGRESSIVE_STAGES": "[]",
+    "FEEDBACK_SCALAR_LR": 0.0005,
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0,
+    "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0,
+    "FEEDBACK_Z_LOSS_START_FRACTION": 1.0,
+    "GLOBAL_BATCH_SIZE": 128,
+    "MICRO_BATCH_SIZE": 8,
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "RUN_NAME": "fbw_L2L11L20_7b_seed42_20260920-023045",
+    "SAVE_INTERVAL": 4000,
+    "SEED": 42,
+    "SEQ_LENGTH": "2048",
+    "TP": "1",
+    "TRAIN_ITERS": 26703,
+    "WANDB_RUN_GROUP": "vanilla_476m_7b_FBW_K3",
+    "WANDB_RUN_ID": "ljsfjz37",
+    "WANDB_TAGS": "product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20"
+  },
+  "eval_passes": 3,
+  "execution_architecture": "full_bandwidth",
+  "feedback_enabled": true,
+  "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+  "feedback_layer_projection_init": "final_layer_identity",
+  "feedback_layer_states": "intermediate block outputs; final after trunk normalization",
+  "feedback_layers": [
+    2,
+    11,
+    20
+  ],
+  "feedback_pass_mixture": {
+    "3": 1.0
+  },
+  "feedback_progressive_stages": [],
+  "ffn_hidden_size": 5632,
+  "final_validation": null,
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "is_final_checkpoint": false,
+  "loading_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.",
+  "logit_softcap": 15.0,
+  "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/fbw_L2L11L20_7b_seed42_20260920-023045",
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "native_inference_adapter": {
+    "active_vocab_size": 32001,
+    "architecture": "full_bandwidth",
+    "context_extension": "native RoPE, no scaling; at most 2x training length",
+    "max_supported_length": 4096,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 0,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "depth_loops": 1,
+      "depth_share_kv": false,
+      "feedback_layers": [
+        2,
+        11,
+        20
+      ],
+      "final_adapter": {
+        "attention_backend": "sdpa_auto",
+        "checkpoint_refiner": true,
+        "ffn_size": null,
+        "kind": "none"
+      },
+      "num_passes": 3
+    },
+    "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled",
+    "step": 8000,
+    "training_context_length": 2048
+  },
+  "native_loading_args": {
+    "architecture": "full_bandwidth",
+    "attention_dropout": 0.0,
+    "aux_mtp_horizon": 1,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "feedback_eval_passes": 3,
+    "feedback_layers": "2,11,20",
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "layernorm_epsilon": 1e-06,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null,
+    "feedback_layer_proj._extra_state": null,
+    "output_layer._extra_state": null
+  },
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "latent_feedback_causal_lm",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "feedback_layer_projection_parameters": 5947392,
+    "feedback_parameters": 3967744,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_trainable": 576059264
+  },
+  "pipeline_parallel_size": 1,
+  "planned_training_tokens": 7000031232,
+  "projection_dtype": "bfloat16 (production)",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 130,
+  "tensor_dtypes": {
+    "torch.bfloat16": 87,
+    "torch.float32": 43
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "total_parameters": 576059264,
+  "total_trainable_parameters": 576059264,
+  "train_passes": 3,
+  "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "training_code_dirty": false,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045",
+  "training_tokens": 2097152000,
+  "vocab_size": 32001,
+  "width_scale": 22
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/launch_config.env b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..e916a214e64499c7ee16f136957057aad090262a
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/launch_config.env
@@ -0,0 +1,107 @@
+# Multi-layer feedback ablation of 476M_7b_FBW0_K3.env: layers 2, 11, 20.
+# See FBW_MULTILAYER_STUDY.md; all training/data settings match the control.
+# See configs/vanilla/FBW_STUDY.md. Derived from 476M_7b.env at
+# a57216c3768aebd1226a30fa573b09530a9537f2; preserve the vanilla global LR curve.
+# Fresh arm: same backbone seed, feedback enabled from update 1; no checkpoint load.
+
+TRAIN_ARCHITECTURE=full_bandwidth
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_vanilla_476m_7b_online_FBW0_K3_L2L11L20
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match the completed vanilla run: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match the completed vanilla data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens.
+# Use the vanilla LR curve from step 0 through the same 7B endpoint.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false
+MEGATRON_EXTRA_ARGS=''
+
+# Use K=3 on every update, from update 1 through update 26,703.
+FEEDBACK_PASS_MIXTURE=3:1.0
+FEEDBACK_PROGRESSIVE_STAGES='[]'
+FEEDBACK_EVAL_PASSES=3
+# 1-based block outputs, with the final layer after trunk normalization.
+# Concatenate 3*1408 -> 1408; initialize projection to [0, 0, I].
+FEEDBACK_LAYERS=2,11,20
+FEEDBACK_LOSS_WEIGHT=1.0
+FEEDBACK_JITTER_SIGMA=0
+FEEDBACK_PREFIX_MIXIN=false
+FEEDBACK_Z_LOSS_COEFFICIENT=0
+FEEDBACK_LOG_HIDDEN_CHANGE=false
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+WANDB_RUN_GROUP=vanilla_476m_7b_FBW_K3
+WANDB_TAGS=product,vanilla,476m,7b,fullbandwidth,pass_k,FBW0,K3,multilayer,L2L11L20
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/manifest.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..b9781f4adb11ff8b0e484283bd0bdc6bf849da89
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/manifest.json
@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 8000,
+  "files": {
+    "README.md": {
+      "sha256": "aab5de6678fc9e11bc6f70fee55ae08a3f7dd392815042ddf025b0f3ba6b41a7",
+      "size": 2696
+    },
+    "config.json": {
+      "sha256": "5ef6a6b687625bc4d8ebabb0eef5da14e72a9133cc48317b27fe086fb945cee6",
+      "size": 12271
+    },
+    "launch_config.env": {
+      "sha256": "637c50c798de3f8796a5112318af3c2ce772a32fd22aa9e2ea11d38b11dccca9",
+      "size": 3606
+    },
+    "model.safetensors": {
+      "sha256": "c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5",
+      "size": 1152254688
+    },
+    "training_recipe.json": {
+      "sha256": "80097382a5141be358f63eb145e5191e0eec2dade0228d6e366eebf394c97db9",
+      "size": 4117
+    }
+  },
+  "manifest_note": "Hashes cover other payload files; manifest does not hash itself.",
+  "path_in_repo": "fbw_L2L11L20_7b_seed42_20260920-023045/step_8000",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "fbw_L2L11L20_7b_seed42_20260920-023045"
+}
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/model.safetensors b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2238131fcb5ddcbc98688ccdd273b943e06ba2d3
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5ce253fab190da088594bcedf0811a687c0d4f203f2ea6083788f8e2b54d2a5
+size 1152254688
diff --git a/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/training_recipe.json b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda730195fd66e7c5d425d357b331b99e3127b07
--- /dev/null
+++ b/FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37/step_8000/training_recipe.json
@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 576059264,
+      "feedback_parameters": 3967744,
+      "feedback_layer_projection_parameters": 5947392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": [],
+    "feedback_layers": [
+      2,
+      11,
+      20
+    ],
+    "feedback_layer_fusion": "concatenate_then_bias_free_linear",
+    "feedback_layer_projection_init": "final_layer_identity",
+    "feedback_layer_states": "intermediate block outputs; final after trunk normalization"
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/NAMING.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/NAMING.json
new file mode 100644
index 0000000000000000000000000000000000000000..88d0342ef8306f8d6fa1046f7f6a47773403b30b
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/NAMING.json
@@ -0,0 +1,10 @@
+{
+  "wandb_id": "qwbqyrvt",
+  "previous_wandb_name": "r4-parent-fbw-b16--qwbqyrvt",
+  "training_run_name": "r4_parent_fbw_b16_20260920-101859",
+  "canonical_name": "FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt",
+  "architecture": "full_bandwidth",
+  "hf_old_paths": [
+    "r4_parent_fbw_b16_20260920-101859"
+  ]
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..da100cb22c085ff27b148990ab8855d559e1209e
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_12000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cee42e38cdd506140efaea15fcfbae1d26fb0fa
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 12000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 1536000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 30976,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 11999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 3145728000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..4cbc62e03a52c8f9e8b33a3607c69d63dc67b5e6
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3184,
+      "sha256": "68ff162efc992c025c73926b107c39fc8986fa49d5197c6ab1b7d5885992ab4d"
+    },
+    "config.json": {
+      "size": 50495,
+      "sha256": "ce8c677682f276b7b64a0b70809ecec0a6420268abb665e53f1f1fa07c9346ab"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "831348a36bcb251f8b896efa57cc06ca6a696a55fbc041485b37d396c62daf9d"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ce1194784ec5bab5e8bdb08625ced9d6e1960168
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:831348a36bcb251f8b896efa57cc06ca6a696a55fbc041485b37d396c62daf9d
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_12000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..97dfbfd9de66eabe0dc60f4f748fc96e2bee1f02
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_16000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 16000, consumed 4,194,304,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e70513a925e473a6bab4c0a1e3a69c300bcb607
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 16000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 2048000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 41216,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 15999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 4194304000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..d1e2ad2941b6702ed454613242769cd2a4040a06
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3184,
+      "sha256": "dfafa0837866bcf6bb7fe653fc6fab1218aa54692aa5259ac87e3697dd4a14f9"
+    },
+    "config.json": {
+      "size": 50495,
+      "sha256": "bc8e421bae01f48e8ef3467a26c6413490ebd962f7357a93d9ea08d7321aeb17"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "2ed99f12c08bb6b532f3f4b5e688ca8b23a7765db65f6fbb33c35adadd2a6aa1"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bbc7908e3423f35294631cfffa28ffe32ea6e896
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed99f12c08bb6b532f3f4b5e688ca8b23a7765db65f6fbb33c35adadd2a6aa1
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_16000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..b13b6277a6390e2ea75edc328049f1a8f43e40d6
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_20000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 20000, consumed 5,242,880,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..c220186bf51152f1a233a73d6d74f1702ec35b2d
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 20000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 2560000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 51456,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 19999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 5242880000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa40e8b70a243ed3bf45d86aef315b2c5cd777a2
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3184,
+      "sha256": "47d8d821eec512cca21840d44e70959909ecabb24a63b5bff9b800261431a11a"
+    },
+    "config.json": {
+      "size": 50495,
+      "sha256": "1bdcf62fdccc9e05a61046bd688ab5b0d714ec803be10229115cf78500043899"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "5eb95f44ffa3e94bb8d2eb6b95d8bac66689c41cdc04b993bfe1ff6bc4b30a9e"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..50309cf26f94bf1fd775eba5d5556751efe18371
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eb95f44ffa3e94bb8d2eb6b95d8bac66689c41cdc04b993bfe1ff6bc4b30a9e
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_20000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..5995afdb3a6b093e9aac5fd8ebc2834b86411a50
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_24000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 24000, consumed 6,291,456,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f2279da828b97fae43e8ec3c45fba2e9f9849cf
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 24000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 3072000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 61696,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 23999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 6291456000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..7891c2e52c9d54b0cbf82cf2a3cb3156977d03f5
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3184,
+      "sha256": "6a83fc83999caa8a13bde0bd3c8b8621abcc4cb137ce2b421875770ec31bf7c7"
+    },
+    "config.json": {
+      "size": 50495,
+      "sha256": "dadd7b770301fb95209f54e9a59c90c2efcb6abf4651a7a08f322f4845ba0cb9"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "3039fa759068863a5f4b9934bbd7da9713b64b076f7ed380a379350e5e967a27"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cf420b6576f3ba493e29f0e0157206f258d16388
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3039fa759068863a5f4b9934bbd7da9713b64b076f7ed380a379350e5e967a27
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_24000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..ca4d91fd2498041c02d1d007f745d2fd026ec86f
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_26703
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..5047a2a46bb73fe6f7fbd32a74b2b99fa662b3f2
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 26703,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 3417984,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 68608,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 26702
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 7000031232,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b0cf64eddcf7d8830d8e12a7bed91320c14ffde
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3184,
+      "sha256": "2b255dfe2342f41e6da06f607c951eb70481b26ddf190619e567c8e18aba885c"
+    },
+    "config.json": {
+      "size": 50495,
+      "sha256": "800af2933ca3fe3015d9a74d3008e9d6605badd63ee2b53b480dba90801cea8f"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "0a57e34d8b0cd134488a8d7d8b409c01c258585f0194aab6281d0a1b36c46ae1"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dae61f18bdb5af5c6c4e5186db2bb1317055dd34
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a57e34d8b0cd134488a8d7d8b409c01c258585f0194aab6281d0a1b36c46ae1
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_26703/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d7c6932ef21cafb61b54fe2e5dac2e8eb1b84eb7
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_4000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 4000, consumed 1,048,576,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c20215e8b7cc8261d7f7eff48b9c4295f586cc4
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 4000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 512000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 10496,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 3999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 1048576000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..bbb45e3fb2f957996c7bd1d764ccac7157d698f4
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3182,
+      "sha256": "bcef7944119409664f3b0e24ae022bceb5d5a48dfa6eaa1ca6e7ab5ede4f83d4"
+    },
+    "config.json": {
+      "size": 50492,
+      "sha256": "41d5bc28251745fd9247fd690c4c59cb2c53a41acda23097312e2a77635b1889"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "76af7449e58be4d0b7ef38c73a2d1cd89b97605d2afbd83374395a560870e4ef"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..37444689a02579dd5131e7a427542ab7f4492794
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76af7449e58be4d0b7ef38c73a2d1cd89b97605d2afbd83374395a560870e4ef
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_4000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/README.md b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..582054a14e0b5c7b233a31cdccb2dc616157dbfe
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/README.md
@@ -0,0 +1,50 @@
+# r4_parent_fbw_b16_20260920-101859 / step_8000
+
+Native full-bandwidth K3 R4 parent model, 570,111,872 parameters. Checkpoint step 8000, consumed 2,097,152,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
+
+Training source is pinned to https://github.com/FFishy-git/loopTF-workspace/tree/9f3c22f21c6026055ba69a39af5f285f09d3cfc8 (repository access may be required). The original training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
+
+Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes through the 20-layer full-bandwidth parent backbone. This parent has no depth-loop adapter or IFM injection.
+
+## Native CPU loading
+
+```python
+import json, torch, importlib, tempfile
+from argparse import Namespace
+from megatron.core import parallel_state
+from megatron.core.process_groups_config import ProcessGroupCollection
+from full_bandwidth_tf.megatron.config import build_model_config
+from full_bandwidth_tf.megatron.builder import FullBandwidthMegatronBuilder
+from safetensors.torch import load_file
+
+def decode(x):
+    if isinstance(x, dict):
+        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
+        if "__dataclass__" in x:
+            module, cls, values = x["__dataclass__"]
+            return getattr(importlib.import_module(module), cls)(**decode(values))
+        if "__enum__" in x:
+            module, cls, member = x["__enum__"]
+            return getattr(importlib.import_module(module), cls)[member]
+        return {k: decode(v) for k, v in x.items()}
+    if isinstance(x, list): return [decode(v) for v in x]
+    return x
+
+c = json.load(open("config.json"))
+a = Namespace(**decode(c["model_arguments"]))
+a.use_cpu_initialization = True
+a.perform_initialization = False
+cfg = build_model_config(a)
+# Standalone CPU loading; use your existing groups in an initialized runtime.
+with tempfile.TemporaryDirectory() as tmp:
+    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
+    parallel_state.initialize_model_parallel()
+    with torch.device("meta"):
+        model = FullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
+    state = load_file("model.safetensors") | c["non_tensor_state"]
+    model.load_state_dict(state, strict=True, assign=True)
+    model.eval()
+    torch.distributed.destroy_process_group()
+```
+
+All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/config.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..4980ad95b5671619eebd1bed06a1537631de0633
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/config.json
@@ -0,0 +1,1339 @@
+{
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "architecture": "full_bandwidth",
+  "backbone_profile": "paper2604_vanilla",
+  "run_name": "r4_parent_fbw_b16_20260920-101859",
+  "iteration": 8000,
+  "checkpoint_version": 3.0,
+  "training_code_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "training_base_revision": "9f3c22f21c6026055ba69a39af5f285f09d3cfc8",
+  "source_provenance": "Training used the pinned training_code_revision with an empty saved code.diff.",
+  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 570111872,
+    "trainable": 570111872
+  },
+  "model_arguments": {
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_size": 1,
+    "sequence_parallel": false,
+    "context_parallel_size": 1,
+    "hierarchical_context_parallel_sizes": null,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "perform_initialization": true,
+    "use_cpu_initialization": null,
+    "fp16": false,
+    "bf16": true,
+    "deterministic_mode": false,
+    "tp_comm_overlap": false,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "cross_entropy_loss_fusion": false,
+    "cross_entropy_fusion_impl": "native",
+    "tp_comm_bootstrap_backend": "nccl",
+    "overlap_moe_expert_parallel_comm": false,
+    "delay_wgrad_compute": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "use_ring_exchange_p2p": false,
+    "defer_embedding_wgrad_compute": false,
+    "wgrad_deferral_limit": 0,
+    "overlap_p2p_comm_warmup_flush": false,
+    "microbatch_group_size_per_vp_stage": null,
+    "mtp_standalone": false,
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "num_layers": 20,
+    "mtp_num_layers": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_use_repeated_layer": false,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "hidden_size": 1408,
+    "num_attention_heads": 11,
+    "attention_backend": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "AttnBackend",
+        "auto"
+      ]
+    },
+    "flash_attention_version": null,
+    "softmax_type": "vanilla",
+    "num_query_groups": 11,
+    "ffn_hidden_size": 5632,
+    "kv_channels": 128,
+    "hidden_dropout": 0.0,
+    "attention_dropout": 0.0,
+    "fp32_residual_connection": false,
+    "apply_residual_connection_post_layernorm": false,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "add_bias_linear": false,
+    "add_qkv_bias": false,
+    "glu_linear_offset": 0.0,
+    "activation_func_clamp_value": null,
+    "rotary_interleaved": false,
+    "normalization": "RMSNorm",
+    "qk_layernorm": false,
+    "qk_l2_norm": false,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "attention_output_gate": false,
+    "calculate_per_token_loss": false,
+    "multi_latent_attention": false,
+    "experimental_attention_variant": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "csa_window_size": 128,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 128,
+    "linear_value_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "init_method_std": 0.02,
+    "embedding_init_method_std": null,
+    "init_model_with_meta_device": false,
+    "use_mup": false,
+    "mup_width_mult": 1.0,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_base_head_dim": null,
+    "mup_attn_scale_power": 1.0,
+    "apply_query_key_layer_scaling": false,
+    "disable_bf16_reduced_precision_matmul": false,
+    "use_fused_weighted_squared_relu": false,
+    "fused_residual_rmsnorm": false,
+    "use_transformer_engine_op_fuser": false,
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_num_layers": null,
+    "distribute_saved_activations": false,
+    "recompute_modules": null,
+    "fp8": null,
+    "fp8_recipe": "delayed",
+    "fp8_quantizer_factory": null,
+    "fp8_margin": 0,
+    "fp8_interval": 1,
+    "fp8_amax_history_len": 1,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_wgrad": true,
+    "fp8_output_proj": false,
+    "first_last_layers_bf16": false,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_at_end_in_bf16": 1,
+    "use_kitchen_attention": false,
+    "kitchen_attention_backend": "sdpa",
+    "fp4": null,
+    "fp4_recipe": "nvfp4",
+    "fp4_quantizer_factory": null,
+    "high_priority_a2a_comm_stream": false,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_overlap": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_ffn_hidden_size": null,
+    "moe_router_topk": 2,
+    "moe_enable_routing_replay": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_num_groups": null,
+    "moe_router_group_topk": null,
+    "moe_router_pre_softmax": false,
+    "moe_router_topk_scaling_factor": null,
+    "moe_router_score_function": "softmax",
+    "moe_router_dtype": null,
+    "moe_router_skip_muon": false,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_force_load_balancing": false,
+    "moe_router_force_biased": null,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "moe_grouped_gemm": false,
+    "moe_use_grouped_tensor": false,
+    "moe_single_grouped_weight": false,
+    "moe_single_grouped_bias": false,
+    "moe_z_loss_coeff": null,
+    "moe_input_jitter_eps": null,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_enable_deepep": false,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_per_layer_logging": false,
+    "moe_expert_capacity_factor": null,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_token_drop_policy": "probs",
+    "moe_layer_recompute": false,
+    "moe_permute_fusion": false,
+    "moe_router_fusion": false,
+    "moe_apply_probs_on_input": false,
+    "moe_latent_size": null,
+    "moe_use_norm_before_up_proj": false,
+    "gtp_remat_opt_in_modules": [],
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_deepep_num_sms": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_ncclep_zero_copy": false,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_mlp_glu_interleave_size": null,
+    "moe_expert_rank_capacity_factor": null,
+    "linear_cp_mode": "headwise",
+    "linear_cp_layout": "zigzag",
+    "attention_cp_layout": "zigzag",
+    "enable_cuda_graph": false,
+    "cuda_graph_warmup_steps": 3,
+    "external_cuda_graph": false,
+    "cuda_graph_impl": "none",
+    "inference_cuda_graph_scope": {
+      "__enum__": [
+        "megatron.core.transformer.enums",
+        "InferenceCudaGraphScope",
+        "none"
+      ]
+    },
+    "enable_mhc_connections": false,
+    "mhc_num_residual_streams": 4,
+    "mhc_sinkhorn_iterations": 20,
+    "mhc_init_gating_factor": 0.01,
+    "use_fused_mhc": false,
+    "mhc_fused_backend": "auto",
+    "mhc_recompute_layer_num": null,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "flash_decode": false,
+    "batch_invariant_mode": false,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "use_te_activation_func": false,
+    "symmetric_ar_type": null,
+    "nccl_all_reduce_for_prefill": false,
+    "inference_fuse_tp_communication": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "mrope_section": null,
+    "is_hybrid_model": false,
+    "mamba_state_dim": 128,
+    "mamba_head_dim": 64,
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "gdp_num_householder": 3,
+    "use_mamba_mem_eff_path": true,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "transformer_impl": "local",
+    "fine_grained_activation_offloading": false,
+    "offload_modules": [],
+    "min_offloaded_tensor_size": 1048576,
+    "delay_offload_until_cuda_graph": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "activation_offload_fraction": 1.0,
+    "moe_paged_stash": false,
+    "moe_paged_stash_page_size": 64,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "encoder_num_layers": 20,
+    "decoder_num_layers": null,
+    "group_query_attention": false,
+    "window_size": null,
+    "window_attn_skip_freq": null,
+    "max_position_embeddings": 2048,
+    "position_embedding_type": "rope",
+    "yarn_original_max_position_embeddings": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "relative_attention_num_buckets": 32,
+    "relative_attention_max_distance": 128,
+    "use_rotary_position_embeddings": false,
+    "rotary_base": 10000,
+    "rotary_percent": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "use_rope_scaling": false,
+    "rope_scaling_factor": 8.0,
+    "no_rope_freq": null,
+    "add_position_embedding": true,
+    "make_vocab_size_divisible_by": 64,
+    "openai_gelu": false,
+    "squared_relu": false,
+    "swiglu": false,
+    "quick_geglu": false,
+    "onnx_safe": null,
+    "bert_binary_head": true,
+    "untie_embeddings_and_output_weights": true,
+    "weight_decay": 0.0,
+    "apply_wd_to_qk_layernorm": false,
+    "clip_grad": 0.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "sgd_momentum": 0.9,
+    "muon_momentum": 0.95,
+    "muon_split_qkv": true,
+    "muon_nesterov": true,
+    "muon_scale_mode": "spectral",
+    "muon_fp32_matmul_prec": "medium",
+    "muon_coefficient_type": "quintic",
+    "muon_num_ns_steps": 5,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "muon_extra_scale_factor": 1.0,
+    "muon_scalar_optimizer": "adam",
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "no_weight_decay_cond_type": null,
+    "profile": false,
+    "profile_step_start": 10,
+    "profile_step_end": 12,
+    "use_pytorch_profiler": false,
+    "pytorch_profiler_collect_shapes": false,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "profile_ranks": [],
+    "record_memory_history": false,
+    "memory_snapshot_path": "snapshot.pickle",
+    "record_shapes": false,
+    "nvtx_ranges": false,
+    "micro_batch_size": 8,
+    "global_batch_size": 128,
+    "rampup_batch_size": null,
+    "step_batch_size_schedule": null,
+    "decrease_batch_size_if_needed": false,
+    "empty_unused_memory_level": 0,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "gpu_sniff_test_interval": null,
+    "train_sync_interval": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "exit_interval": 26703,
+    "exit_duration_in_mins": null,
+    "exit_signal_handler": false,
+    "exit_signal": {
+      "__enum__": [
+        "signal",
+        "Signals",
+        "SIGTERM"
+      ]
+    },
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "manual_gc": false,
+    "manual_gc_interval": 0,
+    "manual_gc_eval": true,
+    "iterations_to_skip": [],
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_large_grads": false,
+    "result_rejected_tracker_filename": null,
+    "tp_comm_overlap_cfg": null,
+    "masked_softmax_fusion": true,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "bias_dropout_fusion": false,
+    "apply_rope_fusion": false,
+    "rope_type": null,
+    "use_flash_attn": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "dump_param_to_param_group_map": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "dataloader_type": "external",
+    "no_persist_layer_norm": false,
+    "gradient_accumulation_fusion": false,
+    "deprecated_use_mcore_models": false,
+    "perform_rl_step": false,
+    "rl_prompts_per_eval": 32,
+    "grpo_prompts_per_step": 32,
+    "grpo_group_size": 2,
+    "rl_generation_lag": null,
+    "rl_max_inflight_requests": null,
+    "rl_submission_granularity": "B",
+    "rl_consumption_granularity": "B",
+    "rl_durable_rollout_bank": false,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "grpo_iterations": 2,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_kl_beta": 0.001,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "langrl_env_config": null,
+    "rl_default_temperature": 1.0,
+    "rl_default_top_p": 0,
+    "rl_default_top_k": -1,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_persist_cuda_graphs": false,
+    "rl_partial_rollouts": false,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_use_sequence_packing": false,
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_training_cuda_graphs": false,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "refit_method": "gloo",
+    "refit_execution_batch_bytes": null,
+    "rl_verify_model_weights_swap": false,
+    "rl_skip_bos_token": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_inference_parsers": [],
+    "seed": 42,
+    "te_rng_tracker": false,
+    "inference_rng_tracker": false,
+    "data_parallel_random_init": false,
+    "init_method_xavier_uniform": false,
+    "lr_decay_style": "linear",
+    "lr_wsd_decay_style": "linear",
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_warmup_fraction": null,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_warmup_init": 0.0,
+    "override_opt_param_scheduler": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "start_weight_decay": 0.0,
+    "end_weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "lr": 0.007470638533717786,
+    "min_lr": 0.0007470638533717787,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "freeze_all_layers": false,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/checkpoints",
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_activations_interval": null,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "save_dgrads_interval": null,
+    "save_retain_interval": 4000,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "non_persistent_save_interval": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "finetune": false,
+    "pretrained_checkpoint": null,
+    "ckpt_step": null,
+    "use_checkpoint_args": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "exit_on_missing_checkpoint": false,
+    "ckpt_format": "torch",
+    "auto_detect_ckpt_format": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_fully_parallel_save": true,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "use_persistent_ckpt_worker": false,
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_assume_constant_structure": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_load_validate_sharding_integrity": true,
+    "strict_fsdp_dtensor_load": true,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "save_tokenizer_assets": true,
+    "replication": false,
+    "replication_jump": null,
+    "replication_factor": 2,
+    "verify_integrity": false,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "override_ckpt_iteration": null,
+    "use_dist_ckpt_deprecated": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_workers": 1,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "grad_reduce_in_bf16": false,
+    "loss_scale": null,
+    "initial_loss_scale": 4294967296,
+    "min_loss_scale": 1.0,
+    "loss_scale_window": 1000,
+    "hysteresis": 2,
+    "attention_softmax_in_fp32": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "fp16_lm_cross_entropy": false,
+    "logit_dtype": null,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "mamba_training_ssm_states_dtype": null,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "align_grad_reduce": true,
+    "local_rank": 0,
+    "lazy_mpu_init": false,
+    "use_megatron_fsdp": false,
+    "use_torch_fsdp2": false,
+    "nccl_communicator_config_path": null,
+    "use_tp_pp_dp_mapping": false,
+    "use_gloo_process_groups": true,
+    "use_sharp": false,
+    "sharp_enabled_group": null,
+    "high_priority_stream_groups": [],
+    "distributed_timeout_seconds_after_init": null,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_trace_buffer_size": 2000,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_extra_dump_on_exec": true,
+    "disable_jit_fuser": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "pipeline_model_parallel_layout": null,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "overlap_p2p_comm": false,
+    "overlap_grad_reduce": false,
+    "ddp_num_buckets": null,
+    "ddp_bucket_size": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_average_in_collective": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "align_param_gather": false,
+    "use_distributed_optimizer": false,
+    "megatron_fsdp_version": 1,
+    "use_layer_wise_param_layout": true,
+    "nccl_ub": false,
+    "disable_symmetric_registration": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_expert_remat_nccl_ub": false,
+    "fsdp_manual_registration": false,
+    "create_all_gather_group": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "expert_data_parallel_sharding_strategy": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "hfsdp_param_gather_overlap": false,
+    "gradient_reduce_div_fusion": true,
+    "fsdp_double_buffer": false,
+    "suggested_communication_unit_size": null,
+    "keep_fp8_transpose_cache": false,
+    "enable_full_sharding_in_hsdp": false,
+    "num_distributed_optimizer_instances": 1,
+    "torch_fsdp2_reshard_after_forward": true,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "max_seqlen_per_dp_cp_rank": null,
+    "hybrid_context_parallel": false,
+    "sequence_packing_scheduler": null,
+    "fake_process_group": false,
+    "eval_iters": 2,
+    "eval_interval": 100,
+    "start_eval_at_iter": null,
+    "eval_global_batch_size": 128,
+    "eval_micro_batch_size": 8,
+    "skip_train": false,
+    "test_mode": false,
+    "full_validation": false,
+    "multiple_validation_sets": false,
+    "validation_set_names": null,
+    "data_path": null,
+    "phase_transition_iterations": null,
+    "split": null,
+    "train_data_path": null,
+    "valid_data_path": null,
+    "test_data_path": null,
+    "data_args_path": null,
+    "per_split_data_args_path": null,
+    "per_dataset_sequences_path": null,
+    "dataloader_fast_cache_load": false,
+    "dataloader_defer_npy_index_mmap": false,
+    "data_cache_path": null,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "seq_length": 2048,
+    "encoder_seq_length": 2048,
+    "decoder_seq_length": null,
+    "sample_rate": 1.0,
+    "mask_prob": 0.15,
+    "short_seq_prob": 0.1,
+    "num_workers": 0,
+    "reset_position_ids": false,
+    "reset_attention_mask": false,
+    "dataloader_inter_document_masking": false,
+    "eod_mask_loss": false,
+    "create_attention_mask_in_dataloader": false,
+    "num_dataset_builder_threads": 1,
+    "object_storage_cache_path": null,
+    "mid_level_dataset_surplus": 0.005,
+    "allow_ambiguous_pad_tokens": false,
+    "fim_data": false,
+    "fim_rate": 0.5,
+    "fim_spm_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_fragment_rate": null,
+    "fim_no_prefix": null,
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_middle_token": "<fim_middle>",
+    "fim_suffix_token": "<fim_suffix>",
+    "fim_pad_token": "<fim_pad>",
+    "fim_eod_token": "<|endoftext|>",
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "pad_vocab_size": true,
+    "vocab_file": null,
+    "merge_file": null,
+    "vocab_extra_ids": 0,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "metadata_path": null,
+    "special_tokens": null,
+    "tiktoken_pattern": null,
+    "tiktoken_num_special_tokens": 1000,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "trust_remote_code": false,
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "chat_template": null,
+    "use_gigatoken": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "ict_head_size": null,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "ict_load": null,
+    "bert_load": null,
+    "titles_data_path": null,
+    "query_in_block_prob": 0.1,
+    "use_one_sent_docs": false,
+    "evidence_data_path": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "block_data_path": null,
+    "embedding_path": null,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "num_classes": 1000,
+    "img_h": 224,
+    "img_w": 224,
+    "num_channels": 3,
+    "patch_dim": 16,
+    "classes_fraction": 1.0,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "head_lr_mult": 1.0,
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vision_backbone_type": "vit",
+    "swin_backbone_type": "tiny",
+    "mask_type": "random",
+    "mask_factor": 1.0,
+    "iter_per_epoch": 1250,
+    "dino_local_img_size": 96,
+    "dino_local_crops_number": 10,
+    "dino_head_hidden_size": 2048,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_norm_last_layer": false,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "num_experts": null,
+    "moe_layer_freq": 1,
+    "moe_use_upcycling": false,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_aux_loss_coeff": 0.0,
+    "moe_upcycling_granularity": 1,
+    "q_lora_rank": null,
+    "kv_lora_rank": 32,
+    "qk_head_dim": 128,
+    "qk_pos_emb_head_dim": 64,
+    "v_head_dim": 128,
+    "rotary_scaling_factor": 1.0,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "cache_mla_latents": false,
+    "mla_down_proj_fusion": false,
+    "linear_attention_freq": null,
+    "heterogeneous_layers_config_path": null,
+    "heterogeneous_layers_config_encoded_json": null,
+    "log_interval": 1,
+    "log_params_norm": false,
+    "log_throughput": false,
+    "log_progress": false,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "log_timers_to_tensorboard": true,
+    "log_loss_scale_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_memory_to_tensorboard": false,
+    "log_memory_interval": null,
+    "log_device_memory_used": false,
+    "log_num_zeros_in_grad": false,
+    "log_max_attention_logit": false,
+    "barrier_with_L1_time": true,
+    "log_world_size_to_tensorboard": false,
+    "wandb_project": "loopTF-workspace",
+    "wandb_exp_name": "r4_parent_fbw_b16_20260920-101859",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/wandb",
+    "wandb_entity": "LoopTF-4-CSPs",
+    "logging_level": null,
+    "log_energy": false,
+    "moe_routing_trace_path": null,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_dump_weights": false,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_load_dir": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_prefetch_factor": 3,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_ignore_errors": false,
+    "log_straggler": false,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "disable_straggler_on_startup": false,
+    "run_workload_inspector_server": false,
+    "inference_batch_times_seqlen_threshold": -1,
+    "max_tokens_to_oom": 12000,
+    "output_bert_embeddings": false,
+    "bert_embedder_type": "megatron",
+    "cuda_graph_modules": [],
+    "use_legacy_static_engine": false,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "decode_only_cuda_graphs": false,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "num_speculative_tokens": 0,
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "offset_sampling_seed_by_dp_rank": true,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_logging_step_interval": 0,
+    "inference_text_gen_server_logging": false,
+    "inference_wandb_logging": false,
+    "inference_coordinator_port": null,
+    "mamba_inference_conv_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "mamba_inference_ssm_states_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_disable_ep_consensus": false,
+    "inference_shards": null,
+    "fp4_param_gather": false,
+    "fp8_param_gather": false,
+    "te_precision_config_file": null,
+    "enable_experimental": false,
+    "spec": null,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "yaml_cfg": null,
+    "use_precision_aware_optimizer": false,
+    "main_grads_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "exp_avg_sq_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_params_dtype": {
+      "__torch_dtype__": "float32"
+    },
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "enable_one_logger": true,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "one_logger_async": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "inprocess_restart": false,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_soft_timeout": 60,
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_last_call_wait": 1,
+    "inprocess_termination_grace_time": 1,
+    "inprocess_granularity": "node",
+    "inprocess_active_world_size": 16,
+    "inprocess_empty_cuda_cache": false,
+    "enable_ft_package": false,
+    "calc_ft_timeouts": false,
+    "ft_num_warmup_iters": 5,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "rerun_mode": "validate_results",
+    "check_for_spiky_loss": false,
+    "enable_msc": false,
+    "disable_msc_deprecated": false,
+    "sft": false,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sft_mock_dataset_config_json": null,
+    "use_varlen_dataset": false,
+    "varlen_sbhd_validation": false,
+    "varlen_mock_dataset_config_json": null,
+    "fault_injector_ranks": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_seed": null,
+    "architecture": "full_bandwidth",
+    "architecture_module": [],
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "training_recipe": "paper2604_21106",
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "paper_reference_width": 640,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_batch_tokens": 262144,
+    "paper_embedding_lr": 0.3,
+    "paper_unembedding_lr": 0.004,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "online_data": true,
+    "online_pack_sequences": false,
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_subset": "sample-100BT",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_source_split": "train",
+    "online_text_column": "text",
+    "online_validation_fraction": 0.001,
+    "online_split_seed": 260421106,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_tokenizer_batch_size": 256,
+    "online_shuffle_buffer_records": 128,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_prefetch_batches": 16,
+    "online_source_mode": "local",
+    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "online_validation_manifest": null,
+    "online_latency_log_dir": null,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/r4_parent_fbw_b16_20260920-101859/dataloader",
+    "eval_at_start": true,
+    "final_eval_iters": 270,
+    "backbone_profile": "paper2604_vanilla",
+    "aux_mtp_horizon": 1,
+    "aux_mtp_rank": 128,
+    "aux_mtp_loss_weight": 0.3,
+    "feedback_loss_weight": 1.0,
+    "feedback_layers": null,
+    "feedback_jitter_sigma": 0.0,
+    "feedback_prefix_mixin": false,
+    "feedback_pass_mixture": "3:1.0",
+    "feedback_progressive_stages": "[]",
+    "feedback_eval_passes": 3,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "feedback_log_hidden_change": false,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_normuon_beta2": 0.95,
+    "adaptive_muon_moment2_method": "normuon",
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "_is_global_batch_size_explicitly_specified": true,
+    "rank": 0,
+    "world_size": 16,
+    "use_dist_ckpt": false,
+    "gtp_weight_remat_size": 1,
+    "transformer_pipeline_model_parallel_size": 1,
+    "data_parallel_size": 16,
+    "virtual_pipeline_model_parallel_size": null,
+    "params_dtype": {
+      "__torch_dtype__": "bfloat16"
+    },
+    "consumed_train_samples": 1024000,
+    "skipped_train_samples": 0,
+    "consumed_valid_samples": 20736,
+    "variable_seq_lengths": false,
+    "expert_gtp_weight_remat_size": 1,
+    "use_layer_wise_distributed_optimizer": false,
+    "recipe_optimizer": {
+      "name": "muonh",
+      "matrix_lr": 0.014,
+      "scalar_lr": null,
+      "matrix_weight_decay": 0.0,
+      "scalar_weight_decay": 0.0,
+      "embedding_lr": 0.3,
+      "unembedding_lr": 0.004,
+      "norm_lr": 0.005,
+      "other_lr": 0.005,
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "eps": 1e-10,
+      "normuon_momentum": 0.95,
+      "normuon_beta2": 0.95,
+      "normuon_nesterov": true,
+      "normuon_adjust_lr": "spectral_norm",
+      "max_grad_norm": null
+    },
+    "recipe_schedule": {
+      "kind": "linear",
+      "warmup_steps": 0,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "z_loss_coefficient": 0.0,
+      "adamc_weight_decay": false
+    },
+    "paper_base_matrix_lr": 0.014,
+    "paper_transfer": {
+      "__dataclass__": [
+        "loop_transformer_backends.recipes",
+        "Paper2604Transfer",
+        {
+          "width": 1408,
+          "total_tokens": 7000031232,
+          "batch_tokens": 262144,
+          "reference_width": 640,
+          "reference_tokens": 983315200,
+          "reference_batch_tokens": 262144
+        }
+      ]
+    },
+    "training_recipe_metadata": {
+      "recipe": "paper2604_21106",
+      "width": 1408,
+      "total_tokens": 7000031232,
+      "batch_tokens": 262144,
+      "reference_width": 640,
+      "reference_tokens": 983315200,
+      "reference_batch_tokens": 262144,
+      "base_lrs": {
+        "matrix": 0.014,
+        "embedding": 0.3,
+        "unembedding": 0.004,
+        "norm": 0.005,
+        "other": 0.005
+      },
+      "peak_lrs": {
+        "matrix": 0.007470638533717786,
+        "embedding": 0.2022599587389726,
+        "unembedding": 0.0026967994498529684,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621
+      },
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "weight_decay": 0.0,
+      "warmup_steps": 0,
+      "final_lr_fraction": 0.1,
+      "optimizer": {
+        "name": "muonh",
+        "matrix_lr": 0.014,
+        "scalar_lr": null,
+        "matrix_weight_decay": 0.0,
+        "scalar_weight_decay": 0.0,
+        "embedding_lr": 0.3,
+        "unembedding_lr": 0.004,
+        "norm_lr": 0.005,
+        "other_lr": 0.005,
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "eps": 1e-10,
+        "normuon_momentum": 0.95,
+        "normuon_beta2": 0.95,
+        "normuon_nesterov": true,
+        "normuon_adjust_lr": "spectral_norm",
+        "max_grad_norm": null
+      },
+      "schedule": {
+        "kind": "linear",
+        "warmup_steps": 0,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "z_loss_coefficient": 0.0,
+        "adamc_weight_decay": false
+      },
+      "group_weight_decay": {
+        "matrix": 0.0,
+        "embedding": 0.0,
+        "unembedding": 0.0,
+        "norm": 0.0,
+        "other": 0.0
+      },
+      "recipe_overrides": {},
+      "requested_batch_tokens": 262144,
+      "sequence_length": 2048,
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "sequence_length": 2048,
+        "round_up": false
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "paper_settings_overridden": false,
+      "model": {
+        "backbone_profile": "paper2604_vanilla",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "width_scale": 22,
+        "hidden_size": 1408,
+        "num_layers": 20,
+        "head_dim": 128,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "ffn_hidden_size": 5632,
+        "activation": "squared_relu",
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "norm_epsilon": 1e-06,
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "bias": false,
+        "dropout": 0.0,
+        "residual_multiplier": 1.0,
+        "residual_and_norm_scale_dtype": "float32",
+        "projection_dtype": "bfloat16 (production)",
+        "tied_embeddings": false,
+        "vocab_size": 32001,
+        "padded_vocab_size": 32064,
+        "logit_softcap": 15.0,
+        "crop_padding_before_loss": true,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "parameter_counts": {
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_trainable": 570111872,
+          "feedback_parameters": 3967744
+        },
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "execution_architecture": "full_bandwidth",
+        "objective": "latent_feedback_causal_lm",
+        "feedback_enabled": true,
+        "train_passes": 3,
+        "eval_passes": 3,
+        "feedback_pass_mixture": {
+          "3": 1.0
+        },
+        "feedback_progressive_stages": []
+      }
+    },
+    "backbone_metadata": {
+      "backbone_profile": "paper2604_vanilla",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "width_scale": 22,
+      "hidden_size": 1408,
+      "num_layers": 20,
+      "head_dim": 128,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "ffn_hidden_size": 5632,
+      "activation": "squared_relu",
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "norm_epsilon": 1e-06,
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "bias": false,
+      "dropout": 0.0,
+      "residual_multiplier": 1.0,
+      "residual_and_norm_scale_dtype": "float32",
+      "projection_dtype": "bfloat16 (production)",
+      "tied_embeddings": false,
+      "vocab_size": 32001,
+      "padded_vocab_size": 32064,
+      "logit_softcap": 15.0,
+      "crop_padding_before_loss": true,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "parameter_counts": {
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_trainable": 570111872,
+        "feedback_parameters": 3967744
+      },
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "execution_architecture": "full_bandwidth",
+      "objective": "latent_feedback_causal_lm",
+      "feedback_enabled": true,
+      "train_passes": 3,
+      "eval_passes": 3,
+      "feedback_pass_mixture": {
+        "3": 1.0
+      },
+      "feedback_progressive_stages": []
+    },
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "initial_iters": 2,
+      "intermediate_iters": 2,
+      "final_validation_iters": 270,
+      "test_iters": 0,
+      "global_batch_size": 128,
+      "sequence_length": 2048,
+      "intermediate_input_tokens": 524288,
+      "final_validation_sequences": 34560,
+      "final_validation_input_tokens": 70778880,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "iteration": 0,
+    "num_floating_point_operations_so_far": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "do_test": 0,
+    "curr_iteration": 7999
+  },
+  "non_tensor_state": {
+    "decoder.layers.0.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.0.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.0.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.1.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.1.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.1.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.2.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.2.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.2.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.3.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.3.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.3.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.4.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.4.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.4.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.5.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.5.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.5.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.6.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.6.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.6.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.7.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.7.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.7.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.8.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.8.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.8.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.9.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.9.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.9.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.10.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.10.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.10.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.11.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.11.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.11.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.12.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.12.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.12.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.13.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.13.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.13.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.14.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.14.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.14.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.15.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.15.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.15.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.16.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.16.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.16.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.17.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.17.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.17.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.18.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.18.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.18.mlp.linear_fc2._extra_state": null,
+    "decoder.layers.19.self_attention.linear_proj._extra_state": null,
+    "decoder.layers.19.self_attention.linear_qkv._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc1._extra_state": null,
+    "decoder.layers.19.mlp.linear_fc2._extra_state": null,
+    "output_layer._extra_state": null,
+    "feedback.state_proj._extra_state": null,
+    "feedback.token_gate_proj._extra_state": null
+  },
+  "tensor_parallel_size": 1,
+  "pipeline_parallel_size": 1,
+  "training_tokens": 2097152000,
+  "training_budget_tokens": 7000031232,
+  "effective_launch_overrides": {
+    "NNODES": 2,
+    "MICRO_BATCH_SIZE": 8,
+    "GLOBAL_BATCH_SIZE": 128,
+    "RUN_NAME": "r4_parent_fbw_b16_20260920-101859",
+    "SAVE_RETAIN_INTERVAL": 4000
+  },
+  "native_source_config": "research/R4_FBW_Architecture_Transition/H1_Midtraining_Transition/configs/parent.env",
+  "tokenizer": {
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "active_vocab_size": 32001,
+    "padded_vocab_size": 32064
+  },
+  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/launch_config.env b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/launch_config.env
new file mode 100644
index 0000000000000000000000000000000000000000..dab31f529558722a12bdd8638e6f05dc6d212e0a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/launch_config.env
@@ -0,0 +1,6 @@
+source "$(dirname "${BASH_SOURCE[0]}")/../../../../configs/vanilla/476M_7b_FBW0_K3.env"
+source "$(dirname "${BASH_SOURCE[0]}")/common.env"
+RUN_NAME=r4_parent_fbw_k3_20l_7b
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,R4,parent,fullbandwidth,K3,20l,7b
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/manifest.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/manifest.json
new file mode 100644
index 0000000000000000000000000000000000000000..62ba13ec59dc213f18e1e7611f61eb29d03eb537
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/manifest.json
@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "size": 3182,
+      "sha256": "438706003edc753de024cf0d0e814d27fbdb2be95728e74643465cc4afe8137e"
+    },
+    "config.json": {
+      "size": 50493,
+      "sha256": "839d3022115046791a5910f37388ead98117cddd94546ed5795031e35147dafd"
+    },
+    "launch_config.env": {
+      "size": 270,
+      "sha256": "b04d4c1529ff04041c587aca71c11a2937af53719c75545b3260559f80b7f59f"
+    },
+    "model.safetensors": {
+      "size": 1140359624,
+      "sha256": "06d0615b440ced88f79e7396171b67c3c280d9e0c521837f1b7ae1127320a61f"
+    },
+    "training_recipe.json": {
+      "size": 3782,
+      "sha256": "263e2b2ddf34ea75952dee46450e9a94c335ca10db7f49b3fede0fb4e8504d4c"
+    }
+  },
+  "note": "Manifest does not hash itself."
+}
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/model.safetensors b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..56eb5006a48c61cf704e43640c58bc45fa10dad7
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d0615b440ced88f79e7396171b67c3c280d9e0c521837f1b7ae1127320a61f
+size 1140359624
diff --git a/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/training_recipe.json b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/training_recipe.json
new file mode 100644
index 0000000000000000000000000000000000000000..04a2f7d863e3e93af62667277e03a2ee73c5605a
--- /dev/null
+++ b/FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt/step_8000/training_recipe.json
@@ -0,0 +1,133 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 570111872,
+      "feedback_parameters": 3967744
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "full_bandwidth",
+    "objective": "latent_feedback_causal_lm",
+    "feedback_enabled": true,
+    "train_passes": 3,
+    "eval_passes": 3,
+    "feedback_pass_mixture": {
+      "3": 1.0
+    },
+    "feedback_progressive_stages": []
+  }
+}
diff --git a/FBW_NAME_MAP.json b/FBW_NAME_MAP.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d3942fb1c061d1ff7462e66ab3f563edf98529f
--- /dev/null
+++ b/FBW_NAME_MAP.json
@@ -0,0 +1,50 @@
+{
+  "schema_version": 1,
+  "pattern": "FBW-K-B-Src-MTP-seed-budget--wandb_id",
+  "runs": [
+    {
+      "wandb_id": "qwbqyrvt",
+      "previous_wandb_name": "r4-parent-fbw-b16--qwbqyrvt",
+      "training_run_name": "r4_parent_fbw_b16_20260920-101859",
+      "canonical_name": "FBW-K3-B20-Src20-MTPOff-seed42-7b--qwbqyrvt",
+      "architecture": "full_bandwidth",
+      "hf_old_paths": [
+        "r4_parent_fbw_b16_20260920-101859"
+      ]
+    },
+    {
+      "wandb_id": "ljsfjz37",
+      "previous_wandb_name": "fbw-layers2-11-20-k3-s42-7b--ljsfjz37",
+      "training_run_name": "fbw_L2L11L20_7b_seed42_20260920-023045",
+      "canonical_name": "FBW-K3-B20-Src2-11-20-MTPOff-seed42-7b--ljsfjz37",
+      "architecture": "full_bandwidth",
+      "hf_old_paths": [
+        "fbw_L2L11L20_7b_seed42_20260920-023045"
+      ]
+    },
+    {
+      "wandb_id": "p4wy8van",
+      "previous_wandb_name": "fbw-23l-k3-7b--p4wy8van",
+      "training_run_name": "paper2604_vanilla_23l_641m_7b_online_FBW0_K3",
+      "canonical_name": "FBW-K3-B23-Src23-MTPOff-seed42-7b--p4wy8van",
+      "architecture": "full_bandwidth",
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "jnd2kygl",
+      "previous_wandb_name": "fbw-mtp4-k3-7b--jnd2kygl",
+      "training_run_name": "paper2604_vanilla_476m_mtp4_7b_online_FBW0_K3",
+      "canonical_name": "FBW-K3-B20-Src20-MTPH4-W0p3-seed42-7b--jnd2kygl",
+      "architecture": "full_bandwidth",
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "h80iubwf",
+      "previous_wandb_name": "fbw-mtp4-k2k3-mix50-7b--h80iubwf",
+      "training_run_name": "paper2604_vanilla_476m_mtp4_7b_online_FBW0_K2K3_50_50",
+      "canonical_name": "FBW-K2p50-K3p50-EvalK3-B20-Src20-MTPH4-W0p3-seed42-7b--h80iubwf",
+      "architecture": "full_bandwidth",
+      "hf_old_paths": []
+    }
+  ]
+}