Add unified Lookahead checkpoint paths and naming map

LOOKAHEAD_NAME_MAP.json

@@ -0,0 +1,429 @@
+{
+  "schema_version": 1,
+  "pattern": "Lookahead-K3-LA2-Aa-Rr-Dd-Pp-Ff-seedN[-distinguishers]--wandb_id",
+  "decimal_separator": "p",
+  "legacy_defaults": "Missing legacy A/R/D/P/F options use the verified backward-compatible A0/R0/0/0/0 policy defaults. Seeds always come from saved W&B configuration.",
+  "runs": [
+    {
+      "wandb_id": "my34axon",
+      "previous_wandb_name": "r4_la_hf_warmstart_s12000_to26703",
+      "training_run_name": "r4_la_hf_warmstart_s12000_to26703",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": true,
+        "normOut": true
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r4_weights_only_d0_p0_a2_r0_s12000_to26703"
+      ]
+    },
+    {
+      "wandb_id": "wkv2n5ub",
+      "previous_wandb_name": "r4_la_hf_warmstart_s20000_to26703",
+      "training_run_name": "r4_la_hf_warmstart_s20000_to26703",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": true,
+        "normOut": true
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r4_weights_only_d0_p0_a2_r0_s20000_to26703"
+      ]
+    },
+    {
+      "wandb_id": "7t0tefok",
+      "previous_wandb_name": "r4_la_hf_warmstart_s16000_to26703",
+      "training_run_name": "r4_la_hf_warmstart_s16000_to26703",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": true,
+        "normOut": true
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r4_weights_only_d0_p0_a2_r0_s16000_to26703"
+      ]
+    },
+    {
+      "wandb_id": "xhm12l4y",
+      "previous_wandb_name": "r3-a1-r0-d0-p0-f0-s42-7b--xhm12l4y",
+      "training_run_name": "r3_D0-P0-A1-R0_7b_seed42_20260920-023045",
+      "canonical_name": "Lookahead-K3-LA2-A1-R0-D0-P0-F0-seed42-7b--xhm12l4y",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A1",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "rsk51lmo",
+      "previous_wandb_name": "r3-a0-r1-d0-p0-f1-s42-7b--rsk51lmo",
+      "training_run_name": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651",
+      "canonical_name": "Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R1",
+        "D": 0,
+        "P": 0,
+        "F": 1,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_D0-P0-A0-R1_7b_seed42_20260919-161651"
+      ]
+    },
+    {
+      "wandb_id": "fob3gqiq",
+      "previous_wandb_name": "r3-a2-r1-d0-p0-f1-s42-7b--fob3gqiq",
+      "training_run_name": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651",
+      "canonical_name": "Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R1",
+        "D": 0,
+        "P": 0,
+        "F": 1,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_D0-P0-A2-R1_7b_seed42_20260919-161651"
+      ]
+    },
+    {
+      "wandb_id": "1jtjk6d7",
+      "previous_wandb_name": "r3-a2-r0-d0-p0-f0-s42-7b--1jtjk6d7",
+      "training_run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_D0-P0-A2-R0_7b_seed42_20260919-160042"
+      ]
+    },
+    {
+      "wandb_id": "gcwdjm01",
+      "previous_wandb_name": "r3-a2-r0-d0-p0p2-f0-s42-7b--gcwdjm01",
+      "training_run_name": "r3_D0-A2-R0_7b_seed42_20260919-040210",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0,
+        "P": 0.2,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_D0-A2-R0_7b_seed42_20260919-040210"
+      ]
+    },
+    {
+      "wandb_id": "3pjop6r1",
+      "previous_wandb_name": "r3-a0-r0-d0p2-p0p2-f0-s42-7b--3pjop6r1",
+      "training_run_name": "r3_D1-A0-R0_7b_seed42_20260919-040211",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0.2,
+        "P": 0.2,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_D1-A0-R0_7b_seed42_20260919-040211"
+      ]
+    },
+    {
+      "wandb_id": "djlht18n",
+      "previous_wandb_name": "r3-a2-r0-d0p2-p0p2-f0-s42-7b--djlht18n",
+      "training_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume",
+      "canonical_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A2",
+        "R": "R0",
+        "D": 0.2,
+        "P": 0.2,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "r3_d1_a2_r0_7b_seed42_20260919-035203-resume"
+      ]
+    },
+    {
+      "wandb_id": "piob4h78",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-gstatesonly-s42-16kstep--piob4h78",
+      "training_run_name": "r3_lookahead_states_only_product_20260918-212932",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-grad-states-only-16kstep--piob4h78",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.05,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "b5xq9efg",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-gheadonly-s42-16kstep--b5xq9efg",
+      "training_run_name": "r3_lookahead_head_only_product_20260918-212932",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-grad-head-only-16kstep--b5xq9efg",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.05,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "7tv5vb6g",
+      "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-14b--7tv5vb6g",
+      "training_run_name": "20260918-LA14B-prod-02",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-14b--7tv5vb6g",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "3qdhqjxj",
+      "previous_wandb_name": "r3-a0-r0-competitive-d0-p0-f0-s42-16kstep--3qdhqjxj",
+      "training_run_name": "20260918-194754_competitive-16gpu-seed42-product",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-fusion-competitive-16kstep--3qdhqjxj",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "db7elm9j",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-s43-16kstep--db7elm9j",
+      "training_run_name": "r3_b300_alpha0p05-seed43_product_20260918-170108",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed43-16kstep--db7elm9j",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.05,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 43,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "bapl3lfs",
+      "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s43-7b--bapl3lfs",
+      "training_run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 43,
+      "hf_old_paths": [
+        "r3_b300_alpha0p0-seed43_product_20260918-170109"
+      ]
+    },
+    {
+      "wandb_id": "fo94dla9",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p2-f0-s42-16kstep--fo94dla9",
+      "training_run_name": "r3_b300_alpha0p2-seed42_product_20260918-170100",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p2-F0-seed42-16kstep--fo94dla9",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.2,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "14vjphny",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p1-f0-s42-16kstep--14vjphny",
+      "training_run_name": "r3_b300_alpha0p1-seed42_product_20260918-165348",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p1-F0-seed42-16kstep--14vjphny",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.1,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "83uufov3",
+      "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-s42-16kstep--83uufov3",
+      "training_run_name": "r3_lookahead_future05_16k_20260918-112504",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-16kstep--83uufov3",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0.05,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "cy5gslq2",
+      "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-16kstep--cy5gslq2",
+      "training_run_name": "r3_lookahead_control_16k_20260918-112311",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-16kstep--cy5gslq2",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": []
+    },
+    {
+      "wandb_id": "9i6byar5",
+      "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-7b--9i6byar5",
+      "training_run_name": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928",
+      "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5",
+      "architecture": "lookahead",
+      "fields": {
+        "K": 3,
+        "A": "A0",
+        "R": "R0",
+        "D": 0,
+        "P": 0,
+        "F": 0,
+        "normIn": false,
+        "normOut": false
+      },
+      "seed": 42,
+      "hf_old_paths": [
+        "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928"
+      ]
+    }
+  ]
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/NAMING.json

@@ -0,0 +1,21 @@
+{
+  "wandb_id": "9i6byar5",
+  "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-7b--9i6byar5",
+  "training_run_name": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928",
+  "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5",
+  "architecture": "lookahead",
+  "fields": {
+    "K": 3,
+    "A": "A0",
+    "R": "R0",
+    "D": 0,
+    "P": 0,
+    "F": 0,
+    "normIn": false,
+    "normOut": false
+  },
+  "seed": 42,
+  "hf_old_paths": [
+    "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928"
+  ]
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/README.md

@@ -0,0 +1,44 @@
+# Lookahead K3 LA2 — step 26703
+
+Training run: `paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928`. Final checkpoint after **7,000,031,232 input tokens**; training exited successfully on 2026-09-18 (Asia/Shanghai).
+
+Native loopTF Lookahead weights, preserving tensor names and dtypes from the TP=1/PP=1 Megatron checkpoint. There are **661,315,072 parameters** in the complete model, including the Lookahead components. K3 uses three complete passes, each with one real and one pseudo backbone forward. LA2 is the existing design's naming label, not two extra pseudo steps.
+
+`model.safetensors`, `config.json`, `launch_config.env`, and `training_recipe.json` follow this repository's vanilla export convention. `config.json` also records constructor arguments and final validation; `manifest.json` records payload sizes and SHA256 hashes. This is a model-weights export, not a complete training-resume checkpoint. Optimizer, scheduler, RNG, dataloader state and credentials are excluded.
+
+## Load with the project implementation
+
+Use [loopTF-workspace](https://github.com/FFishy-git/loopTF-workspace) at training revision `432ef3046b5cf834ad4f61e3985cd3013343eeba` with its `src` directory on `PYTHONPATH` and the dependencies installed. This custom architecture is **not** a Transformers `AutoModel.from_pretrained` export.
+
+After downloading this step folder, set `export_dir` to its local path:
+
+```python
+import json
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+
+export_dir = Path("/path/to/step_26703")
+config = json.loads((export_dir / "config.json").read_text())
+with torch.device("meta"):
+    model = LookaheadTransformer(
+        ModelConfig(**config["model_config"]),
+        LookaheadConfig(**config["lookahead_config"]),
+    )
+state = load_file(str(export_dir / "model.safetensors"), device="cpu")
+assert all(key.startswith("model.") for key in state)
+model.load_state_dict(
+    {key.removeprefix("model."): value for key, value in state.items()},
+    strict=True, assign=True,
+)
+model.eval()
+```
+
+Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` described in `config.json` (32,001 active tokens). The vocabulary tensors retain padding to 32,064. The launch snapshot records base settings; `training_run` in `config.json` is the effective timestamped run identity.
+
+Final held-out pass-3 CE: **2.27749586**, perplexity **9.75222887**, over 34,560 sequences / 70,778,880 input tokens. No new evaluation or training was performed during export.
+
+[Weights & Biases run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/9i6byar5)

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/config.json

@@ -0,0 +1,140 @@
+{
+  "activation": "squared_relu",
+  "activation_checkpointing": false,
+  "attention": "full_causal",
+  "attention_output_gate": false,
+  "backbone_profile": "paper2604_vanilla",
+  "bias": false,
+  "block_application_multiplier": 6.449999999999999,
+  "cache_layer_indices": [
+    0,
+    10,
+    19
+  ],
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 26703,
+  "checkpoint_refiner": false,
+  "checkpoint_version": 3.0,
+  "crop_padding_before_loss": true,
+  "dropout": 0.0,
+  "eval_passes": 3,
+  "execution_architecture": "lookahead",
+  "feedback_loss_weight": 1.0,
+  "ffn_hidden_size": 5632,
+  "final_validation": {
+    "eval/checkpoint_step": 26703,
+    "eval/final_val/dataset_id": "config-a71f5512f13ffd6f",
+    "eval/final_val/final_ce": 2.277495861053467,
+    "eval/final_val/final_ppl": 9.752228871149752,
+    "eval/final_val/num_input_tokens": 70778880,
+    "eval/final_val/num_sequences": 34560,
+    "eval/final_val/num_tokens": 70778880,
+    "eval/final_val/pass_1_ce": 2.3000683784484863,
+    "eval/final_val/pass_1_ppl": 9.974864497254137,
+    "eval/final_val/pass_2_ce": 2.2785534858703613,
+    "eval/final_val/pass_2_ppl": 9.762548526623844,
+    "eval/final_val/pass_3_ce": 2.277495861053467,
+    "eval/final_val/pass_3_ppl": 9.752228871149752,
+    "eval/final_val/pass_count": 3,
+    "eval/is_final": true
+  },
+  "head_dim": 128,
+  "hidden_size": 1408,
+  "initialization": {
+    "embedding_normal_std": 1.0,
+    "head_normal_std": 0.001,
+    "matrix_uniform_bound": "sqrt(3/fan_in)",
+    "norm_scale": 1.0
+  },
+  "loading_note": "Native LookaheadMegatronModel state_dict keys, not a Transformers AutoModel export. For LookaheadTransformer, remove exactly the leading model. prefix and load with strict=True; assign=True preserves saved dtypes. No optimizer, scheduler, RNG, or data-loader state is included.",
+  "logit_softcap": 15.0,
+  "lookahead_config": {
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "feedback_loss_weight": 1.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632
+  },
+  "lookahead_mode": "parallel",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "model_level_norms": [
+    "embedding",
+    "recur",
+    "final"
+  ],
+  "naming_note": "K3 is three complete passes including the seed. LA2 labels the existing one-real-plus-one-pseudo-forward design per pass; it does not add two future pseudo steps.",
+  "non_tensor_state_dict_entries": {},
+  "norm_epsilon": 1e-06,
+  "num_attention_heads": 11,
+  "num_kv_heads": 11,
+  "num_layers": 20,
+  "objective": "lookahead_parallel_next_token",
+  "padded_vocab_size": 32064,
+  "parameter_counts": {
+    "backbone_total_trainable": 566144128,
+    "feedback_parameters": 3964928,
+    "lookahead_parameters": 95170944,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_parameters": 661315072,
+    "total_trainable": 661315072
+  },
+  "pipeline_parallel_size": 1,
+  "projection_dtype": "bfloat16 (production)",
+  "pseudo_backend": "sdpa_auto",
+  "qk_norm": "functional_rmsnorm_after_rope",
+  "refiner_activation": "silu_gated",
+  "residual_and_norm_scale_dtype": "float32",
+  "residual_multiplier": 1.0,
+  "rope_base": 10000.0,
+  "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+  "sequence_length": 2048,
+  "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+  "tensor_count": 194,
+  "tensor_dtypes": {
+    "torch.bfloat16": 143,
+    "torch.float32": 51
+  },
+  "tensor_parallel_size": 1,
+  "tied_embeddings": false,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "additional_token": "<|assistant_end|>",
+    "repo_id": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+  "train_passes": 3,
+  "training_code_revision": "432ef3046b5cf834ad4f61e3985cd3013343eeba",
+  "training_run": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928",
+  "training_tokens": 7000031232,
+  "vocab_size": 32001,
+  "width_scale": 22
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/launch_config.env

@@ -0,0 +1,100 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=false # Profiled on 32 B300 GPUs; avoids recompute with ~101.4 GiB peak allocation/GPU.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=false
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/manifest.json

@@ -0,0 +1,30 @@
+{
+  "checkpoint_iteration": 26703,
+  "files": {
+    "README.md": {
+      "sha256": "3b876c301ac9b01783e3cae3a5c0247e496f73175d7302de1e3566beb90b9d58",
+      "size": 2744
+    },
+    "config.json": {
+      "sha256": "64c12b7b17ce62c57bf71f69c6223fad2f4ad33d190fbe67976b333b3c51faab",
+      "size": 4984
+    },
+    "launch_config.env": {
+      "sha256": "4939b5b392c71bc4111cb30f954903da4b5296160589603c8fe9aaae3b988d59",
+      "size": 3365
+    },
+    "model.safetensors": {
+      "sha256": "47c9c162f943bde560d66e39809b2e70f27e7c72f5bb117ec96194176d489c83",
+      "size": 1322796200
+    },
+    "training_recipe.json": {
+      "sha256": "7688bd955bc0d6fb1b134a5f92ab88d000a87a67f0cd051f079445b2e3b74130",
+      "size": 4092
+    }
+  },
+  "manifest_note": "Hashes cover the other payload files; this manifest does not hash itself.",
+  "path_in_repo": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928/step_26703",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "training_run": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c9c162f943bde560d66e39809b2e70f27e7c72f5bb117ec96194176d489c83
+size 1322796200

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/training_recipe.json

@@ -0,0 +1,142 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 661315072,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 95170944,
+      "total_parameters": 661315072
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999
+  }
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/NAMING.json

@@ -0,0 +1,21 @@
+{
+  "wandb_id": "bapl3lfs",
+  "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s43-7b--bapl3lfs",
+  "training_run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs",
+  "architecture": "lookahead",
+  "fields": {
+    "K": 3,
+    "A": "A0",
+    "R": "R0",
+    "D": 0,
+    "P": 0,
+    "F": 0,
+    "normIn": false,
+    "normOut": false
+  },
+  "seed": 43,
+  "hf_old_paths": [
+    "r3_b300_alpha0p0-seed43_product_20260918-170109"
+  ]
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/README.md

@@ -0,0 +1,7 @@
+# r3_b300_alpha0p0-seed43_product_20260918-170109: native resumable checkpoint at step12000
+
+Private training-state backup, not a Transformers model export. Training was cancelled by direct user approval. Last logged completed update was13933;1933 subsequent-to-checkpoint logged updates are unsaved. This bundle covers step12000 only, not later logged progress.
+
+Preserves native model/optimizer/scheduler/RNG and eight DP-rank loader states. Restore with matching source7e264fdbf158f9a8f41cb6d09b29ed71b867ca27 plus code.diff and source_overlay, Megatron d046b0de593b1802c78a7664765174c031eb452c, same TP1/PP1/DP8 and pinned tokenizer/data. Source checkout access is required; dataset is excluded. Review launch_config.env and resume_provenance.json. Point LOAD_PATH to the restored checkpoints/ directory, DATALOADER_SAVE to the restored dataloader/ directory, RESUME_POLICY=auto, REQUIRE_CHECKPOINT=true, WANDB_RUN_ID=bapl3lfs, WANDB_RESUME=must and preserve product/descriptive tags plus resume. Keep original26703 LR horizon and16000 total-step cap. Paths in the frozen config must be relocated explicitly as needed. No resume launch is authorized by this backup.
+
+No execution test of restoration performed. Checksum manifest excludes itself; immutable remote verification is pending. Never load untrusted native pickle files.

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ba9446ef7ad5452940570bdf51ed3d0668c6f60476a459b1fd3215229c4b2ae
+size 6974760755

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/latest_checkpointed_iteration.txt

@@ -0,0 +1 @@
+12000

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/code.diff

@@ -0,0 +1,14 @@
+diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py
+index fc03aaf..8047ea2 100644
+--- a/src/loop_transformer_backends/megatron/runtime.py
++++ b/src/loop_transformer_backends/megatron/runtime.py
+@@ -175,6 +175,9 @@ def run(
+             )
+         model_config = architecture.build_model_config(args)
+         with ExitStack() as hooks:
++            if writer is not None:
++                from .wandb_storage import install_wandb_file_hashing
++                hooks.callback(install_wandb_file_hashing())
+             hooks.callback(install_ft_args_hook(args))
+             if args.training_recipe == DEFAULT_RECIPE:
+                 hooks.enter_context(paper_checkpoint_context())

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2de254e0312563219685177629aaf79a70aad8ebf933963916c025a19cbffcd3
+size 6665781

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7039b2866ebc92a8ff3cc124ad1ea212565909d6135c84f70882164e1293bea3
+size 6762549

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92f0fac9c5ee1f888ca73800dd487f6bd5f5e5cf61527f17aff72c83978592a
+size 6338485

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23d668390740b3bc9067316c2c06e4854fab1e8b9a8808ef49603abcbb552606
+size 6927349

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15715a11d4e1dade60eed8965fc7061d0492330e4b6a2b1b1f04ee61dae44c6d
+size 6409205

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfa876b9239b76f5aa13c77f85b1183627337d058f8b13bc2a90e9f7478ac65c
+size 6878517

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55d4dad7cc4b3481f750dcb4d4ba6f1c7ee03fe13628454efc15b274e854b5f3
+size 6970293

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2346d6fe631dadf7b6694c9863a182214bafaa6527c3dd01dd620aff0a7e8783
+size 6647861

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/launch_config.env

@@ -0,0 +1,123 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=true
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
+
+# Pinned R3 single-node objective-strength experiment. No architecture changes.
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0
+SEED=43
+NNODES=1
+GPUS_PER_NODE=8
+TP=1
+PP=1
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+TRAIN_ITERS=26703
+EXIT_INTERVAL=16000
+SAVE_INTERVAL=4000
+SAVE_RETAIN_INTERVAL=4000
+ENABLE_RECOMPUTE=false
+LOOKAHEAD_CHECKPOINT_REFINER=false
+EVAL_AT_START=true
+FINAL_EVAL_ITERS=270
+LOG_INTERVAL=1
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300
+WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/manifest.json

@@ -0,0 +1,77 @@
+{
+  "repo_id": "Siyuc/loopTF-workspace",
+  "path_in_repo": "r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000",
+  "format": "native_megatron_full_training_resume",
+  "saved_step": 12000,
+  "files": {
+    "README.md": {
+      "bytes": 1334,
+      "sha256": "d2e6647b39d233d5086c1df9208336dc3c4fe0bd689af9dbef5051b3f877ad19"
+    },
+    "checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt": {
+      "bytes": 6974760755,
+      "sha256": "1ba9446ef7ad5452940570bdf51ed3d0668c6f60476a459b1fd3215229c4b2ae"
+    },
+    "checkpoints/latest_checkpointed_iteration.txt": {
+      "bytes": 6,
+      "sha256": "cc9f23a77ecea60b29786c934ac6c3f8fbe766405b3dc3f2b4b02679be56ba88"
+    },
+    "code.diff": {
+      "bytes": 745,
+      "sha256": "09aec79ad20ab40065f952858ac9ce6cfe9f7590ae35646d28c13a9b7e1dad0c"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt": {
+      "bytes": 6665781,
+      "sha256": "2de254e0312563219685177629aaf79a70aad8ebf933963916c025a19cbffcd3"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt": {
+      "bytes": 6762549,
+      "sha256": "7039b2866ebc92a8ff3cc124ad1ea212565909d6135c84f70882164e1293bea3"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt": {
+      "bytes": 6338485,
+      "sha256": "c92f0fac9c5ee1f888ca73800dd487f6bd5f5e5cf61527f17aff72c83978592a"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt": {
+      "bytes": 6927349,
+      "sha256": "23d668390740b3bc9067316c2c06e4854fab1e8b9a8808ef49603abcbb552606"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt": {
+      "bytes": 6409205,
+      "sha256": "15715a11d4e1dade60eed8965fc7061d0492330e4b6a2b1b1f04ee61dae44c6d"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt": {
+      "bytes": 6878517,
+      "sha256": "bfa876b9239b76f5aa13c77f85b1183627337d058f8b13bc2a90e9f7478ac65c"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt": {
+      "bytes": 6970293,
+      "sha256": "55d4dad7cc4b3481f750dcb4d4ba6f1c7ee03fe13628454efc15b274e854b5f3"
+    },
+    "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt": {
+      "bytes": 6647861,
+      "sha256": "2346d6fe631dadf7b6694c9863a182214bafaa6527c3dd01dd620aff0a7e8783"
+    },
+    "launch_config.env": {
+      "bytes": 3907,
+      "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26"
+    },
+    "project_revision.txt": {
+      "bytes": 41,
+      "sha256": "d9cf81883983241a3d6f60c54a46ce386c0c2db114ae17e8ad5fb67dc27795f2"
+    },
+    "resume_provenance.json": {
+      "bytes": 2483,
+      "sha256": "c00d741cde09d95839669c88d50cc79c86876fd624608e7eb71cf29bcfa544cc"
+    },
+    "source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py": {
+      "bytes": 672,
+      "sha256": "976b04f0d3a471acb4542167795f7d17ebc897c533b1f1ebc93c53e4cb01ffff"
+    },
+    "training_recipe.json": {
+      "bytes": 4123,
+      "sha256": "7a62b1c7ab13f0b5d7d3b05de1eab2b338813322a885cb9f8564fe1c1e7189fd"
+    }
+  },
+  "manifest_excludes_itself": true
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/project_revision.txt

@@ -0,0 +1 @@
+7e264fdbf158f9a8f41cb6d09b29ed71b867ca27

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/resume_provenance.json

@@ -0,0 +1,94 @@
+{
+  "format": "native_megatron_full_training_resume",
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "saved_step": 12000,
+  "cancelled": true,
+  "cancellation_status": "user_cancelled_verified",
+  "last_logged_step_at_preparation": 13812,
+  "source_revision": "7e264fdbf158f9a8f41cb6d09b29ed71b867ca27",
+  "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c",
+  "alpha": 0.0,
+  "seed": 43,
+  "tp": 1,
+  "pp": 1,
+  "dp": 8,
+  "components": {
+    "model": true,
+    "optimizer": true,
+    "opt_param_scheduler": true,
+    "rng_state": true
+  },
+  "rng_state_count": 1,
+  "loader_states": [
+    {
+      "rank": 0,
+      "bytes": 6665781,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 1,
+      "bytes": 6762549,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 2,
+      "bytes": 6338485,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 3,
+      "bytes": 6927349,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 4,
+      "bytes": 6409205,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 5,
+      "bytes": 6878517,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 6,
+      "bytes": 6970293,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    },
+    {
+      "rank": 7,
+      "bytes": 6647861,
+      "top_level_keys": [
+        "dataloader_state_dict"
+      ]
+    }
+  ],
+  "wandb_id": "bapl3lfs",
+  "lr_horizon": 26703,
+  "total_executed_step_cap": 16000,
+  "missing_state": [],
+  "limitations": [
+    "Native trusted-pickle format; load only trusted checkpoints with pinned source/dependencies.",
+    "No training-resume execution performed.",
+    "Only saved-step state is backed up; later optimizer updates are not part of this payload.",
+    "Source code repository access and pinned dataset/tokenizer are required; training data is excluded."
+  ],
+  "rng_state_note": "One shared native RNG record is expected: data_parallel_random_init=False. Pinned Megatron checkpointing.py saves one record and restores index0 for each DP rank. All eight distinct data-loader states are included.",
+  "stop_step": 13933,
+  "stop_step_definition": "Last fully logged completed optimizer update; an interrupted in-flight update is not claimed.",
+  "unsaved_logged_updates": 1933
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py

@@ -0,0 +1,18 @@
+"""Buffered W&B file hashing for shared checkpoint filesystems."""
+from __future__ import annotations
+
+def _buffered_file_hasher(hasher, *paths):
+    for path in sorted(map(str, paths)):
+        with open(path, "rb") as handle:
+            for chunk in iter(lambda: handle.read(128 * 1024), b""):
+                hasher.update(chunk)
+    return hasher
+
+def install_wandb_file_hashing():
+    """Avoid fatal mmap faults; retain MD5/xxHash and sorted-path contracts."""
+    from wandb.sdk.lib import hashutil
+    original = hashutil._file_hasher
+    hashutil._file_hasher = _buffered_file_hasher
+    def restore():
+        hashutil._file_hasher = original
+    return restore

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/training_recipe.json

@@ -0,0 +1,143 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 661315072,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 95170944,
+      "total_parameters": 661315072
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999,
+    "future_loss_weight": 0.0
+  }
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/README.md

@@ -0,0 +1,40 @@
+# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 16000
+
+Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment.
+Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint.
+This checkpoint consumed 4194304000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable.
+The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle.
+
+Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769
+Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132.
+W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs
+Access to the code repository is required. Paper backbone source revision is separately recorded in config.json.
+
+## Loading
+
+Use the pinned repository and its dependencies, with its src directory on PYTHONPATH.
+These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion.
+
+```python
+import json, torch
+from safetensors.torch import load_file
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+
+cfg = json.load(open("config.json"))
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**cfg["model_config"]),
+                                 LookaheadConfig(**cfg["lookahead_config"]))
+state = load_file("model.safetensors")
+state.update(cfg["non_tensor_state"])
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k[len("model."):]: v for k, v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass.
+The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments.
+Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export.
+No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700.

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/config.json

@@ -0,0 +1,237 @@
+{
+  "architecture": "lookahead",
+  "backbone_profile": "paper2604_vanilla",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_version": 3.0,
+  "data_parallel_size": 8,
+  "effective_launch_settings": {
+    "ADAM_BETA1": "0.8",
+    "ADAM_BETA2": "0.95",
+    "ADAM_EPS": "1e-10",
+    "ATTENTION_DROPOUT": "0",
+    "ATTENTION_OUTPUT_GATE": "false",
+    "AUX_MTP_HORIZON": "1",
+    "AUX_MTP_LOSS_WEIGHT": "0.3",
+    "AUX_MTP_RANK": "128",
+    "BACKBONE_PROFILE": "paper2604_vanilla",
+    "CKPT_FORMAT": "torch",
+    "CLIP_GRAD": "0.0",
+    "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader",
+    "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_PREFLIGHT": "false",
+    "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data",
+    "DISABLE_BIAS_LINEAR": "true",
+    "DISABLE_WANDB": "false",
+    "ENABLE_RECOMPUTE": "false",
+    "EVAL_AT_START": "true",
+    "EVAL_INTERVAL": "100",
+    "EVAL_ITERS": "2",
+    "EVAL_ONLY": "false",
+    "EXIT_INTERVAL": "26703",
+    "EXIT_SIGNAL_HANDLER": "false",
+    "EXPECTED_INPUT_LENGTH": "2048",
+    "EXPECTED_RECORD_LENGTH": "2049",
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": "0.95",
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": "0.0005",
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0",
+    "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0",
+    "FEEDBACK_Z_LOSS_START_FRACTION": "1.0",
+    "FFN_HIDDEN_SIZE": "5632",
+    "FINAL_EVAL_ITERS": "270",
+    "GLOBAL_BATCH_SIZE": "128",
+    "GPUS_PER_NODE": "8",
+    "GROUP_QUERY_ATTENTION": "false",
+    "HIDDEN_DROPOUT": "0",
+    "HIDDEN_SIZE": "1408",
+    "LAYERNORM_EPSILON": "0.000001",
+    "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints",
+    "LOG_INTERVAL": "1",
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CHECKPOINT_REFINER": "false",
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1",
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0",
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": "3",
+    "LOOKAHEAD_REFINER_FFN_SIZE": "5632",
+    "LOOKAHEAD_REFINER_GATE_BIAS": "-2",
+    "LR": "0.014",
+    "LR_DECAY_STYLE": "linear",
+    "LR_WSD_DECAY_STYLE": "linear",
+    "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64",
+    "MAX_POSITION_EMBEDDINGS": "2048",
+    "MEGATRON_EXTRA_ARGS": "",
+    "MICRO_BATCH_SIZE": "4",
+    "MIN_LR": "0.0007470638533717787",
+    "MIN_MODEL_INPUT_TOKENS": "7000000000",
+    "MUON_MOMENTUM": "0.95",
+    "MUON_NESTEROV": "true",
+    "NORMALIZATION": "RMSNorm",
+    "NUM_ATTENTION_HEADS": "11",
+    "NUM_LAYERS": "20",
+    "NUM_QUERY_GROUPS": "11",
+    "NUM_WORKERS": "0",
+    "ONLINE_DATA": "true",
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "ONLINE_LATENCY_LOG_DIR": "",
+    "ONLINE_PACK_SEQUENCES": "false",
+    "ONLINE_PREFETCH_BATCHES": "16",
+    "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "ONLINE_SHUFFLE_BUFFER_RECORDS": "128",
+    "ONLINE_SOURCE_MODE": "local",
+    "ONLINE_SOURCE_RETRY_LIMIT": "8",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TEXT_COLUMN": "text",
+    "ONLINE_TOKENIZER_BATCH_SIZE": "256",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "ONLINE_VALIDATION_MANIFEST": "",
+    "OPTIMIZER": "muonh",
+    "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output",
+    "PAPER_EMBEDDING_LR": "0.3",
+    "PAPER_NORM_LR": "0.005",
+    "PAPER_OTHER_LR": "0.005",
+    "PAPER_REFERENCE_BATCH_TOKENS": "262144",
+    "PAPER_REFERENCE_TOKENS": "983315200",
+    "PAPER_REFERENCE_WIDTH": "640",
+    "PAPER_UNEMBEDDING_LR": "0.004",
+    "POSITION_EMBEDDING_TYPE": "rope",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "QK_LAYERNORM": "false",
+    "RECIPE_OVERRIDES": "{}",
+    "RECIPE_REQUESTED_BATCH_TOKENS": "262144",
+    "RECOMPUTE_GRANULARITY": "full",
+    "RECOMPUTE_METHOD": "uniform",
+    "RECOMPUTE_NUM_LAYERS": "1",
+    "REQUIRE_CHECKPOINT": "true",
+    "RESUME_POLICY": "auto",
+    "ROTARY_BASE": "10000",
+    "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847",
+    "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "SAVE_INTERVAL": "4000",
+    "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints",
+    "SAVE_RETAIN_INTERVAL": "4000",
+    "SEED": "43",
+    "SEQ_LENGTH": "2048",
+    "SWIGLU": "false",
+    "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard",
+    "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "TOKENIZER_TYPE": "HuggingFaceTokenizer",
+    "TP": "1",
+    "TRAINING_RECIPE": "paper2604_21106",
+    "TRAIN_ARCHITECTURE": "lookahead",
+    "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document",
+    "TRAIN_ITERS": "26703",
+    "UNTIE_EMBEDDINGS": "true",
+    "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "VOCAB_SIZE": "32001",
+    "WANDB_ENTITY": "LoopTF-4-CSPs",
+    "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "WANDB_MODE": "online",
+    "WANDB_PROJECT": "loopTF-workspace",
+    "WANDB_RESUME": "must",
+    "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918",
+    "WANDB_RUN_ID": "bapl3lfs",
+    "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb",
+    "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200",
+    "WARMUP_ITERS": "0",
+    "WEIGHT_DECAY": "0.0",
+    "WINDOW_ATTN_SKIP_FREQ": "1",
+    "WINDOW_SIZE": "none",
+    "WSD_DECAY_ITERS": "0"
+  },
+  "iteration": 16000,
+  "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.",
+  "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.",
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R0"
+  },
+  "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "non_tensor_state": {},
+  "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 661315072,
+    "trainable": 661315072
+  },
+  "pipeline_parallel_size": 1,
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "tensor_parallel_size": 1,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "padded_vocab_size": 32064,
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "training": {
+    "completed_run": true,
+    "consumed_input_tokens": 4194304000,
+    "consumed_train_samples": 2048000,
+    "final_checkpoint": false,
+    "global_batch_size": 128,
+    "micro_batch_size": 4,
+    "planned_input_tokens": 7000031232,
+    "resumed_from_step": 12000,
+    "seed": 43,
+    "sequence_length": 2048,
+    "train_iters": 26703
+  },
+  "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace",
+  "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/launch_config.env

@@ -0,0 +1,123 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=true
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
+
+# Pinned R3 single-node objective-strength experiment. No architecture changes.
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0
+SEED=43
+NNODES=1
+GPUS_PER_NODE=8
+TP=1
+PP=1
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+TRAIN_ITERS=26703
+EXIT_INTERVAL=16000
+SAVE_INTERVAL=4000
+SAVE_RETAIN_INTERVAL=4000
+ENABLE_RECOMPUTE=false
+LOOKAHEAD_CHECKPOINT_REFINER=false
+EVAL_AT_START=true
+FINAL_EVAL_ITERS=270
+LOG_INTERVAL=1
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300
+WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    {
+      "path": "README.md",
+      "sha256": "2759094b2b3e6649ec56b17b2674144884f6cba9a2114c0997539c729daa6e48",
+      "size": 2352
+    },
+    {
+      "path": "config.json",
+      "sha256": "15826e3ae8ecfaa74cc08baeaf202aecfef75ce20d1f60db11d9b51a235c319b",
+      "size": 10068
+    },
+    {
+      "path": "launch_config.env",
+      "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26",
+      "size": 3907
+    },
+    {
+      "path": "model.safetensors",
+      "sha256": "4d66347d78097750130c50e659f02a134eed43c90fd1e6d8779c8788795998e9",
+      "size": 1322796152
+    },
+    {
+      "path": "training_recipe.json",
+      "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6",
+      "size": 4352
+    }
+  ],
+  "format": "safetensors_native_megatron_state_dict",
+  "iteration": 16000,
+  "note": "This manifest does not hash itself.",
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d66347d78097750130c50e659f02a134eed43c90fd1e6d8779c8788795998e9
+size 1322796152

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/training_recipe.json

@@ -0,0 +1,150 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 661315072,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 95170944,
+      "total_parameters": 661315072
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999,
+    "future_loss_weight": 0.0,
+    "future_gradient_mode": "all",
+    "draft_loss_weight": 0.0,
+    "pseudo_initializer": "A0",
+    "pseudo_top_k": 128,
+    "pseudo_temperature": 1.0,
+    "refiner_variant": "R0",
+    "allow_objective_change_on_resume": false
+  }
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/README.md

@@ -0,0 +1,40 @@
+# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 20000
+
+Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment.
+Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint.
+This checkpoint consumed 5242880000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable.
+The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle.
+
+Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769
+Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132.
+W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs
+Access to the code repository is required. Paper backbone source revision is separately recorded in config.json.
+
+## Loading
+
+Use the pinned repository and its dependencies, with its src directory on PYTHONPATH.
+These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion.
+
+```python
+import json, torch
+from safetensors.torch import load_file
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+
+cfg = json.load(open("config.json"))
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**cfg["model_config"]),
+                                 LookaheadConfig(**cfg["lookahead_config"]))
+state = load_file("model.safetensors")
+state.update(cfg["non_tensor_state"])
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k[len("model."):]: v for k, v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass.
+The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments.
+Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export.
+No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700.

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/config.json

@@ -0,0 +1,237 @@
+{
+  "architecture": "lookahead",
+  "backbone_profile": "paper2604_vanilla",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_version": 3.0,
+  "data_parallel_size": 8,
+  "effective_launch_settings": {
+    "ADAM_BETA1": "0.8",
+    "ADAM_BETA2": "0.95",
+    "ADAM_EPS": "1e-10",
+    "ATTENTION_DROPOUT": "0",
+    "ATTENTION_OUTPUT_GATE": "false",
+    "AUX_MTP_HORIZON": "1",
+    "AUX_MTP_LOSS_WEIGHT": "0.3",
+    "AUX_MTP_RANK": "128",
+    "BACKBONE_PROFILE": "paper2604_vanilla",
+    "CKPT_FORMAT": "torch",
+    "CLIP_GRAD": "0.0",
+    "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader",
+    "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_PREFLIGHT": "false",
+    "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data",
+    "DISABLE_BIAS_LINEAR": "true",
+    "DISABLE_WANDB": "false",
+    "ENABLE_RECOMPUTE": "false",
+    "EVAL_AT_START": "true",
+    "EVAL_INTERVAL": "100",
+    "EVAL_ITERS": "2",
+    "EVAL_ONLY": "false",
+    "EXIT_INTERVAL": "26703",
+    "EXIT_SIGNAL_HANDLER": "false",
+    "EXPECTED_INPUT_LENGTH": "2048",
+    "EXPECTED_RECORD_LENGTH": "2049",
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": "0.95",
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": "0.0005",
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0",
+    "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0",
+    "FEEDBACK_Z_LOSS_START_FRACTION": "1.0",
+    "FFN_HIDDEN_SIZE": "5632",
+    "FINAL_EVAL_ITERS": "270",
+    "GLOBAL_BATCH_SIZE": "128",
+    "GPUS_PER_NODE": "8",
+    "GROUP_QUERY_ATTENTION": "false",
+    "HIDDEN_DROPOUT": "0",
+    "HIDDEN_SIZE": "1408",
+    "LAYERNORM_EPSILON": "0.000001",
+    "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints",
+    "LOG_INTERVAL": "1",
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CHECKPOINT_REFINER": "false",
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1",
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0",
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": "3",
+    "LOOKAHEAD_REFINER_FFN_SIZE": "5632",
+    "LOOKAHEAD_REFINER_GATE_BIAS": "-2",
+    "LR": "0.014",
+    "LR_DECAY_STYLE": "linear",
+    "LR_WSD_DECAY_STYLE": "linear",
+    "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64",
+    "MAX_POSITION_EMBEDDINGS": "2048",
+    "MEGATRON_EXTRA_ARGS": "",
+    "MICRO_BATCH_SIZE": "4",
+    "MIN_LR": "0.0007470638533717787",
+    "MIN_MODEL_INPUT_TOKENS": "7000000000",
+    "MUON_MOMENTUM": "0.95",
+    "MUON_NESTEROV": "true",
+    "NORMALIZATION": "RMSNorm",
+    "NUM_ATTENTION_HEADS": "11",
+    "NUM_LAYERS": "20",
+    "NUM_QUERY_GROUPS": "11",
+    "NUM_WORKERS": "0",
+    "ONLINE_DATA": "true",
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "ONLINE_LATENCY_LOG_DIR": "",
+    "ONLINE_PACK_SEQUENCES": "false",
+    "ONLINE_PREFETCH_BATCHES": "16",
+    "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "ONLINE_SHUFFLE_BUFFER_RECORDS": "128",
+    "ONLINE_SOURCE_MODE": "local",
+    "ONLINE_SOURCE_RETRY_LIMIT": "8",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TEXT_COLUMN": "text",
+    "ONLINE_TOKENIZER_BATCH_SIZE": "256",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "ONLINE_VALIDATION_MANIFEST": "",
+    "OPTIMIZER": "muonh",
+    "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output",
+    "PAPER_EMBEDDING_LR": "0.3",
+    "PAPER_NORM_LR": "0.005",
+    "PAPER_OTHER_LR": "0.005",
+    "PAPER_REFERENCE_BATCH_TOKENS": "262144",
+    "PAPER_REFERENCE_TOKENS": "983315200",
+    "PAPER_REFERENCE_WIDTH": "640",
+    "PAPER_UNEMBEDDING_LR": "0.004",
+    "POSITION_EMBEDDING_TYPE": "rope",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "QK_LAYERNORM": "false",
+    "RECIPE_OVERRIDES": "{}",
+    "RECIPE_REQUESTED_BATCH_TOKENS": "262144",
+    "RECOMPUTE_GRANULARITY": "full",
+    "RECOMPUTE_METHOD": "uniform",
+    "RECOMPUTE_NUM_LAYERS": "1",
+    "REQUIRE_CHECKPOINT": "true",
+    "RESUME_POLICY": "auto",
+    "ROTARY_BASE": "10000",
+    "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847",
+    "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "SAVE_INTERVAL": "4000",
+    "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints",
+    "SAVE_RETAIN_INTERVAL": "4000",
+    "SEED": "43",
+    "SEQ_LENGTH": "2048",
+    "SWIGLU": "false",
+    "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard",
+    "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "TOKENIZER_TYPE": "HuggingFaceTokenizer",
+    "TP": "1",
+    "TRAINING_RECIPE": "paper2604_21106",
+    "TRAIN_ARCHITECTURE": "lookahead",
+    "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document",
+    "TRAIN_ITERS": "26703",
+    "UNTIE_EMBEDDINGS": "true",
+    "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "VOCAB_SIZE": "32001",
+    "WANDB_ENTITY": "LoopTF-4-CSPs",
+    "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "WANDB_MODE": "online",
+    "WANDB_PROJECT": "loopTF-workspace",
+    "WANDB_RESUME": "must",
+    "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918",
+    "WANDB_RUN_ID": "bapl3lfs",
+    "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb",
+    "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200",
+    "WARMUP_ITERS": "0",
+    "WEIGHT_DECAY": "0.0",
+    "WINDOW_ATTN_SKIP_FREQ": "1",
+    "WINDOW_SIZE": "none",
+    "WSD_DECAY_ITERS": "0"
+  },
+  "iteration": 20000,
+  "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.",
+  "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.",
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R0"
+  },
+  "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "non_tensor_state": {},
+  "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 661315072,
+    "trainable": 661315072
+  },
+  "pipeline_parallel_size": 1,
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "tensor_parallel_size": 1,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "padded_vocab_size": 32064,
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "training": {
+    "completed_run": true,
+    "consumed_input_tokens": 5242880000,
+    "consumed_train_samples": 2560000,
+    "final_checkpoint": false,
+    "global_batch_size": 128,
+    "micro_batch_size": 4,
+    "planned_input_tokens": 7000031232,
+    "resumed_from_step": 12000,
+    "seed": 43,
+    "sequence_length": 2048,
+    "train_iters": 26703
+  },
+  "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace",
+  "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/launch_config.env

@@ -0,0 +1,123 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=true
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
+
+# Pinned R3 single-node objective-strength experiment. No architecture changes.
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0
+SEED=43
+NNODES=1
+GPUS_PER_NODE=8
+TP=1
+PP=1
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+TRAIN_ITERS=26703
+EXIT_INTERVAL=16000
+SAVE_INTERVAL=4000
+SAVE_RETAIN_INTERVAL=4000
+ENABLE_RECOMPUTE=false
+LOOKAHEAD_CHECKPOINT_REFINER=false
+EVAL_AT_START=true
+FINAL_EVAL_ITERS=270
+LOG_INTERVAL=1
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300
+WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    {
+      "path": "README.md",
+      "sha256": "306b3f2228014d8fabf82bcd4365ffefc3b43db8a2796f3c44d9f9997a9d9f28",
+      "size": 2352
+    },
+    {
+      "path": "config.json",
+      "sha256": "a6be34edaa2940c5ca6fca659333ffeb081347394208c841b0bc95f8f08b3e40",
+      "size": 10068
+    },
+    {
+      "path": "launch_config.env",
+      "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26",
+      "size": 3907
+    },
+    {
+      "path": "model.safetensors",
+      "sha256": "9a9789ee2c00ea0bfa583a11b86d8f4cb48bc7821a6cdf907048823acaee1176",
+      "size": 1322796152
+    },
+    {
+      "path": "training_recipe.json",
+      "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6",
+      "size": 4352
+    }
+  ],
+  "format": "safetensors_native_megatron_state_dict",
+  "iteration": 20000,
+  "note": "This manifest does not hash itself.",
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9789ee2c00ea0bfa583a11b86d8f4cb48bc7821a6cdf907048823acaee1176
+size 1322796152

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/training_recipe.json

@@ -0,0 +1,150 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 661315072,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 95170944,
+      "total_parameters": 661315072
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999,
+    "future_loss_weight": 0.0,
+    "future_gradient_mode": "all",
+    "draft_loss_weight": 0.0,
+    "pseudo_initializer": "A0",
+    "pseudo_top_k": 128,
+    "pseudo_temperature": 1.0,
+    "refiner_variant": "R0",
+    "allow_objective_change_on_resume": false
+  }
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/README.md

@@ -0,0 +1,40 @@
+# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 24000
+
+Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment.
+Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint.
+This checkpoint consumed 6291456000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable.
+The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle.
+
+Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769
+Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132.
+W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs
+Access to the code repository is required. Paper backbone source revision is separately recorded in config.json.
+
+## Loading
+
+Use the pinned repository and its dependencies, with its src directory on PYTHONPATH.
+These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion.
+
+```python
+import json, torch
+from safetensors.torch import load_file
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+
+cfg = json.load(open("config.json"))
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**cfg["model_config"]),
+                                 LookaheadConfig(**cfg["lookahead_config"]))
+state = load_file("model.safetensors")
+state.update(cfg["non_tensor_state"])
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k[len("model."):]: v for k, v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass.
+The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments.
+Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export.
+No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700.

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/config.json

@@ -0,0 +1,237 @@
+{
+  "architecture": "lookahead",
+  "backbone_profile": "paper2604_vanilla",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_version": 3.0,
+  "data_parallel_size": 8,
+  "effective_launch_settings": {
+    "ADAM_BETA1": "0.8",
+    "ADAM_BETA2": "0.95",
+    "ADAM_EPS": "1e-10",
+    "ATTENTION_DROPOUT": "0",
+    "ATTENTION_OUTPUT_GATE": "false",
+    "AUX_MTP_HORIZON": "1",
+    "AUX_MTP_LOSS_WEIGHT": "0.3",
+    "AUX_MTP_RANK": "128",
+    "BACKBONE_PROFILE": "paper2604_vanilla",
+    "CKPT_FORMAT": "torch",
+    "CLIP_GRAD": "0.0",
+    "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader",
+    "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_PREFLIGHT": "false",
+    "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data",
+    "DISABLE_BIAS_LINEAR": "true",
+    "DISABLE_WANDB": "false",
+    "ENABLE_RECOMPUTE": "false",
+    "EVAL_AT_START": "true",
+    "EVAL_INTERVAL": "100",
+    "EVAL_ITERS": "2",
+    "EVAL_ONLY": "false",
+    "EXIT_INTERVAL": "26703",
+    "EXIT_SIGNAL_HANDLER": "false",
+    "EXPECTED_INPUT_LENGTH": "2048",
+    "EXPECTED_RECORD_LENGTH": "2049",
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": "0.95",
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": "0.0005",
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0",
+    "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0",
+    "FEEDBACK_Z_LOSS_START_FRACTION": "1.0",
+    "FFN_HIDDEN_SIZE": "5632",
+    "FINAL_EVAL_ITERS": "270",
+    "GLOBAL_BATCH_SIZE": "128",
+    "GPUS_PER_NODE": "8",
+    "GROUP_QUERY_ATTENTION": "false",
+    "HIDDEN_DROPOUT": "0",
+    "HIDDEN_SIZE": "1408",
+    "LAYERNORM_EPSILON": "0.000001",
+    "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints",
+    "LOG_INTERVAL": "1",
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CHECKPOINT_REFINER": "false",
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1",
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0",
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": "3",
+    "LOOKAHEAD_REFINER_FFN_SIZE": "5632",
+    "LOOKAHEAD_REFINER_GATE_BIAS": "-2",
+    "LR": "0.014",
+    "LR_DECAY_STYLE": "linear",
+    "LR_WSD_DECAY_STYLE": "linear",
+    "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64",
+    "MAX_POSITION_EMBEDDINGS": "2048",
+    "MEGATRON_EXTRA_ARGS": "",
+    "MICRO_BATCH_SIZE": "4",
+    "MIN_LR": "0.0007470638533717787",
+    "MIN_MODEL_INPUT_TOKENS": "7000000000",
+    "MUON_MOMENTUM": "0.95",
+    "MUON_NESTEROV": "true",
+    "NORMALIZATION": "RMSNorm",
+    "NUM_ATTENTION_HEADS": "11",
+    "NUM_LAYERS": "20",
+    "NUM_QUERY_GROUPS": "11",
+    "NUM_WORKERS": "0",
+    "ONLINE_DATA": "true",
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "ONLINE_LATENCY_LOG_DIR": "",
+    "ONLINE_PACK_SEQUENCES": "false",
+    "ONLINE_PREFETCH_BATCHES": "16",
+    "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "ONLINE_SHUFFLE_BUFFER_RECORDS": "128",
+    "ONLINE_SOURCE_MODE": "local",
+    "ONLINE_SOURCE_RETRY_LIMIT": "8",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TEXT_COLUMN": "text",
+    "ONLINE_TOKENIZER_BATCH_SIZE": "256",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "ONLINE_VALIDATION_MANIFEST": "",
+    "OPTIMIZER": "muonh",
+    "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output",
+    "PAPER_EMBEDDING_LR": "0.3",
+    "PAPER_NORM_LR": "0.005",
+    "PAPER_OTHER_LR": "0.005",
+    "PAPER_REFERENCE_BATCH_TOKENS": "262144",
+    "PAPER_REFERENCE_TOKENS": "983315200",
+    "PAPER_REFERENCE_WIDTH": "640",
+    "PAPER_UNEMBEDDING_LR": "0.004",
+    "POSITION_EMBEDDING_TYPE": "rope",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "QK_LAYERNORM": "false",
+    "RECIPE_OVERRIDES": "{}",
+    "RECIPE_REQUESTED_BATCH_TOKENS": "262144",
+    "RECOMPUTE_GRANULARITY": "full",
+    "RECOMPUTE_METHOD": "uniform",
+    "RECOMPUTE_NUM_LAYERS": "1",
+    "REQUIRE_CHECKPOINT": "true",
+    "RESUME_POLICY": "auto",
+    "ROTARY_BASE": "10000",
+    "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847",
+    "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "SAVE_INTERVAL": "4000",
+    "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints",
+    "SAVE_RETAIN_INTERVAL": "4000",
+    "SEED": "43",
+    "SEQ_LENGTH": "2048",
+    "SWIGLU": "false",
+    "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard",
+    "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "TOKENIZER_TYPE": "HuggingFaceTokenizer",
+    "TP": "1",
+    "TRAINING_RECIPE": "paper2604_21106",
+    "TRAIN_ARCHITECTURE": "lookahead",
+    "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document",
+    "TRAIN_ITERS": "26703",
+    "UNTIE_EMBEDDINGS": "true",
+    "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "VOCAB_SIZE": "32001",
+    "WANDB_ENTITY": "LoopTF-4-CSPs",
+    "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "WANDB_MODE": "online",
+    "WANDB_PROJECT": "loopTF-workspace",
+    "WANDB_RESUME": "must",
+    "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918",
+    "WANDB_RUN_ID": "bapl3lfs",
+    "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb",
+    "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200",
+    "WARMUP_ITERS": "0",
+    "WEIGHT_DECAY": "0.0",
+    "WINDOW_ATTN_SKIP_FREQ": "1",
+    "WINDOW_SIZE": "none",
+    "WSD_DECAY_ITERS": "0"
+  },
+  "iteration": 24000,
+  "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.",
+  "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.",
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R0"
+  },
+  "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "non_tensor_state": {},
+  "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 661315072,
+    "trainable": 661315072
+  },
+  "pipeline_parallel_size": 1,
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "tensor_parallel_size": 1,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "padded_vocab_size": 32064,
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "training": {
+    "completed_run": true,
+    "consumed_input_tokens": 6291456000,
+    "consumed_train_samples": 3072000,
+    "final_checkpoint": false,
+    "global_batch_size": 128,
+    "micro_batch_size": 4,
+    "planned_input_tokens": 7000031232,
+    "resumed_from_step": 12000,
+    "seed": 43,
+    "sequence_length": 2048,
+    "train_iters": 26703
+  },
+  "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace",
+  "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/launch_config.env

@@ -0,0 +1,123 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=true
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
+
+# Pinned R3 single-node objective-strength experiment. No architecture changes.
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0
+SEED=43
+NNODES=1
+GPUS_PER_NODE=8
+TP=1
+PP=1
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+TRAIN_ITERS=26703
+EXIT_INTERVAL=16000
+SAVE_INTERVAL=4000
+SAVE_RETAIN_INTERVAL=4000
+ENABLE_RECOMPUTE=false
+LOOKAHEAD_CHECKPOINT_REFINER=false
+EVAL_AT_START=true
+FINAL_EVAL_ITERS=270
+LOG_INTERVAL=1
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300
+WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    {
+      "path": "README.md",
+      "sha256": "4f3f67f7cd0fbcff1a600f749e19594a449c14f16d26a275d95d0792579627b7",
+      "size": 2352
+    },
+    {
+      "path": "config.json",
+      "sha256": "40aa54a4e499638a7a319a26923e2d2d1ffb881ad97ef32417ff450d96682693",
+      "size": 10068
+    },
+    {
+      "path": "launch_config.env",
+      "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26",
+      "size": 3907
+    },
+    {
+      "path": "model.safetensors",
+      "sha256": "d66f32b66871e65a15bee0dade243f1c77c5924f2732e0afe75a11b25a15fb3b",
+      "size": 1322796152
+    },
+    {
+      "path": "training_recipe.json",
+      "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6",
+      "size": 4352
+    }
+  ],
+  "format": "safetensors_native_megatron_state_dict",
+  "iteration": 24000,
+  "note": "This manifest does not hash itself.",
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d66f32b66871e65a15bee0dade243f1c77c5924f2732e0afe75a11b25a15fb3b
+size 1322796152

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/training_recipe.json

@@ -0,0 +1,150 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 661315072,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 95170944,
+      "total_parameters": 661315072
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999,
+    "future_loss_weight": 0.0,
+    "future_gradient_mode": "all",
+    "draft_loss_weight": 0.0,
+    "pseudo_initializer": "A0",
+    "pseudo_top_k": 128,
+    "pseudo_temperature": 1.0,
+    "refiner_variant": "R0",
+    "allow_objective_change_on_resume": false
+  }
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/README.md

@@ -0,0 +1,40 @@
+# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 26703
+
+Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment.
+Training finished at update 26703 (7000031232 input tokens); this checkpoint is the final checkpoint.
+This checkpoint consumed 7000031232 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable.
+The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle.
+
+Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769
+Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132.
+W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs
+Access to the code repository is required. Paper backbone source revision is separately recorded in config.json.
+
+## Loading
+
+Use the pinned repository and its dependencies, with its src directory on PYTHONPATH.
+These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion.
+
+```python
+import json, torch
+from safetensors.torch import load_file
+from lookahead_tf.hf.config import ModelConfig
+from lookahead_tf.hf.model import LookaheadTransformer
+from lookahead_tf.policy import LookaheadConfig
+
+cfg = json.load(open("config.json"))
+with torch.device("meta"):
+    model = LookaheadTransformer(ModelConfig(**cfg["model_config"]),
+                                 LookaheadConfig(**cfg["lookahead_config"]))
+state = load_file("model.safetensors")
+state.update(cfg["non_tensor_state"])
+assert all(k.startswith("model.") for k in state)
+model.load_state_dict({k[len("model."):]: v for k, v in state.items()},
+                      strict=True, assign=True)
+model.eval()
+```
+
+K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass.
+The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments.
+Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export.
+No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700.

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/config.json

@@ -0,0 +1,237 @@
+{
+  "architecture": "lookahead",
+  "backbone_profile": "paper2604_vanilla",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_version": 3.0,
+  "data_parallel_size": 8,
+  "effective_launch_settings": {
+    "ADAM_BETA1": "0.8",
+    "ADAM_BETA2": "0.95",
+    "ADAM_EPS": "1e-10",
+    "ATTENTION_DROPOUT": "0",
+    "ATTENTION_OUTPUT_GATE": "false",
+    "AUX_MTP_HORIZON": "1",
+    "AUX_MTP_LOSS_WEIGHT": "0.3",
+    "AUX_MTP_RANK": "128",
+    "BACKBONE_PROFILE": "paper2604_vanilla",
+    "CKPT_FORMAT": "torch",
+    "CLIP_GRAD": "0.0",
+    "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader",
+    "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049",
+    "DATA_PREFLIGHT": "false",
+    "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data",
+    "DISABLE_BIAS_LINEAR": "true",
+    "DISABLE_WANDB": "false",
+    "ENABLE_RECOMPUTE": "false",
+    "EVAL_AT_START": "true",
+    "EVAL_INTERVAL": "100",
+    "EVAL_ITERS": "2",
+    "EVAL_ONLY": "false",
+    "EXIT_INTERVAL": "26703",
+    "EXIT_SIGNAL_HANDLER": "false",
+    "EXPECTED_INPUT_LENGTH": "2048",
+    "EXPECTED_RECORD_LENGTH": "2049",
+    "FEEDBACK_EVAL_PASSES": "",
+    "FEEDBACK_JITTER_SIGMA": "0.02",
+    "FEEDBACK_LOG_HIDDEN_CHANGE": "false",
+    "FEEDBACK_LOSS_WEIGHT": "1.0",
+    "FEEDBACK_NORMUON_BETA2": "0.95",
+    "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03",
+    "FEEDBACK_PREFIX_MIXIN": "true",
+    "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]",
+    "FEEDBACK_SCALAR_LR": "0.0005",
+    "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0",
+    "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0",
+    "FEEDBACK_Z_LOSS_START_FRACTION": "1.0",
+    "FFN_HIDDEN_SIZE": "5632",
+    "FINAL_EVAL_ITERS": "270",
+    "GLOBAL_BATCH_SIZE": "128",
+    "GPUS_PER_NODE": "8",
+    "GROUP_QUERY_ATTENTION": "false",
+    "HIDDEN_DROPOUT": "0",
+    "HIDDEN_SIZE": "1408",
+    "LAYERNORM_EPSILON": "0.000001",
+    "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints",
+    "LOG_INTERVAL": "1",
+    "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto",
+    "LOOKAHEAD_CHECKPOINT_REFINER": "false",
+    "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1",
+    "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all",
+    "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0",
+    "LOOKAHEAD_MODE": "parallel",
+    "LOOKAHEAD_NUM_PASSES": "3",
+    "LOOKAHEAD_REFINER_FFN_SIZE": "5632",
+    "LOOKAHEAD_REFINER_GATE_BIAS": "-2",
+    "LR": "0.014",
+    "LR_DECAY_STYLE": "linear",
+    "LR_WSD_DECAY_STYLE": "linear",
+    "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64",
+    "MAX_POSITION_EMBEDDINGS": "2048",
+    "MEGATRON_EXTRA_ARGS": "",
+    "MICRO_BATCH_SIZE": "4",
+    "MIN_LR": "0.0007470638533717787",
+    "MIN_MODEL_INPUT_TOKENS": "7000000000",
+    "MUON_MOMENTUM": "0.95",
+    "MUON_NESTEROV": "true",
+    "NORMALIZATION": "RMSNorm",
+    "NUM_ATTENTION_HEADS": "11",
+    "NUM_LAYERS": "20",
+    "NUM_QUERY_GROUPS": "11",
+    "NUM_WORKERS": "0",
+    "ONLINE_DATA": "true",
+    "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu",
+    "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "ONLINE_DATASET_SUBSET": "sample-100BT",
+    "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "ONLINE_LATENCY_LOG_DIR": "",
+    "ONLINE_PACK_SEQUENCES": "false",
+    "ONLINE_PREFETCH_BATCHES": "16",
+    "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
+    "ONLINE_SHUFFLE_BUFFER_RECORDS": "128",
+    "ONLINE_SOURCE_MODE": "local",
+    "ONLINE_SOURCE_RETRY_LIMIT": "8",
+    "ONLINE_SOURCE_SPLIT": "train",
+    "ONLINE_SPLIT_SEED": "260421106",
+    "ONLINE_TEXT_COLUMN": "text",
+    "ONLINE_TOKENIZER_BATCH_SIZE": "256",
+    "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "ONLINE_VALIDATION_FRACTION": "0.001",
+    "ONLINE_VALIDATION_MANIFEST": "",
+    "OPTIMIZER": "muonh",
+    "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output",
+    "PAPER_EMBEDDING_LR": "0.3",
+    "PAPER_NORM_LR": "0.005",
+    "PAPER_OTHER_LR": "0.005",
+    "PAPER_REFERENCE_BATCH_TOKENS": "262144",
+    "PAPER_REFERENCE_TOKENS": "983315200",
+    "PAPER_REFERENCE_WIDTH": "640",
+    "PAPER_UNEMBEDDING_LR": "0.004",
+    "POSITION_EMBEDDING_TYPE": "rope",
+    "PP": "1",
+    "PRECISION": "bf16",
+    "QK_LAYERNORM": "false",
+    "RECIPE_OVERRIDES": "{}",
+    "RECIPE_REQUESTED_BATCH_TOKENS": "262144",
+    "RECOMPUTE_GRANULARITY": "full",
+    "RECOMPUTE_METHOD": "uniform",
+    "RECOMPUTE_NUM_LAYERS": "1",
+    "REQUIRE_CHECKPOINT": "true",
+    "RESUME_POLICY": "auto",
+    "ROTARY_BASE": "10000",
+    "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847",
+    "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "SAVE_INTERVAL": "4000",
+    "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints",
+    "SAVE_RETAIN_INTERVAL": "4000",
+    "SEED": "43",
+    "SEQ_LENGTH": "2048",
+    "SWIGLU": "false",
+    "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard",
+    "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "TOKENIZER_TYPE": "HuggingFaceTokenizer",
+    "TP": "1",
+    "TRAINING_RECIPE": "paper2604_21106",
+    "TRAIN_ARCHITECTURE": "lookahead",
+    "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document",
+    "TRAIN_ITERS": "26703",
+    "UNTIE_EMBEDDINGS": "true",
+    "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document",
+    "VOCAB_SIZE": "32001",
+    "WANDB_ENTITY": "LoopTF-4-CSPs",
+    "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+    "WANDB_MODE": "online",
+    "WANDB_PROJECT": "loopTF-workspace",
+    "WANDB_RESUME": "must",
+    "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918",
+    "WANDB_RUN_ID": "bapl3lfs",
+    "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb",
+    "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200",
+    "WARMUP_ITERS": "0",
+    "WEIGHT_DECAY": "0.0",
+    "WINDOW_ATTN_SKIP_FREQ": "1",
+    "WINDOW_SIZE": "none",
+    "WSD_DECAY_ITERS": "0"
+  },
+  "iteration": 26703,
+  "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.",
+  "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.",
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "num_passes": 3,
+    "pseudo_initializer": "A0",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_variant": "R0"
+  },
+  "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c",
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "non_tensor_state": {},
+  "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
+  "parameter_counts": {
+    "total": 661315072,
+    "trainable": 661315072
+  },
+  "pipeline_parallel_size": 1,
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109",
+  "tensor_parallel_size": 1,
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "padded_vocab_size": 32064,
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "training": {
+    "completed_run": true,
+    "consumed_input_tokens": 7000031232,
+    "consumed_train_samples": 3417984,
+    "final_checkpoint": true,
+    "global_batch_size": 128,
+    "micro_batch_size": 4,
+    "planned_input_tokens": 7000031232,
+    "resumed_from_step": 12000,
+    "seed": 43,
+    "sequence_length": 2048,
+    "train_iters": 26703
+  },
+  "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace",
+  "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/launch_config.env

@@ -0,0 +1,123 @@
+# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe.
+# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass.
+# See configs/lookahead/README.md for the additional pseudo/refiner computation.
+TRAIN_ARCHITECTURE=lookahead
+BACKBONE_PROFILE=paper2604_vanilla
+TRAINING_RECIPE=paper2604_21106
+RECIPE_OVERRIDES='{}'
+# Use the installed Transformers release; the cluster's dev checkout is incompatible.
+TRANSFORMERS_PATH=''
+RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+
+# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed.
+# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT.
+DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049
+ONLINE_DATA=true
+# Match FBW0_K3: use the complete local raw cache when available.
+ONLINE_SOURCE_MODE=auto
+ONLINE_DATASET=HuggingFaceFW/fineweb-edu
+ONLINE_DATASET_SUBSET=sample-100BT
+ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9
+ONLINE_SOURCE_SPLIT=train
+ONLINE_TEXT_COLUMN=text
+ONLINE_VALIDATION_FRACTION=0.001
+ONLINE_SPLIT_SEED=260421106
+TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf
+ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d
+ONLINE_TOKENIZER_BATCH_SIZE=256
+ONLINE_SHUFFLE_BUFFER_RECORDS=128
+NUM_WORKERS=0
+DATA_PREFLIGHT=false
+EXPECTED_RECORD_LENGTH=2049
+EXPECTED_INPUT_LENGTH=2048
+MIN_MODEL_INPUT_TOKENS=7000000000
+
+# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32.
+# The multi-node launcher supplies the node rank; preflight can override NNODES.
+NNODES=4
+GPUS_PER_NODE=8
+TP=1
+PP=1
+# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1).
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+SEQ_LENGTH=2048
+MAX_POSITION_EMBEDDINGS=2048
+# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens.
+TRAIN_ITERS=26703
+
+NUM_LAYERS=20
+HIDDEN_SIZE=1408
+FFN_HIDDEN_SIZE=5632
+NUM_ATTENTION_HEADS=11
+NUM_QUERY_GROUPS=11
+GROUP_QUERY_ATTENTION=false
+ATTENTION_OUTPUT_GATE=false
+QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE.
+SWIGLU=false       # Profile supplies squared-ReLU.
+DISABLE_BIAS_LINEAR=true
+NORMALIZATION=RMSNorm
+LAYERNORM_EPSILON=0.000001
+HIDDEN_DROPOUT=0
+ATTENTION_DROPOUT=0
+WINDOW_SIZE=none
+WINDOW_ATTN_SKIP_FREQ=1
+POSITION_EMBEDDING_TYPE=rope
+ROTARY_BASE=10000
+UNTIE_EMBEDDINGS=true
+TOKENIZER_TYPE=HuggingFaceTokenizer
+VOCAB_SIZE=32001
+MAKE_VOCAB_SIZE_DIVISIBLE_BY=64
+PRECISION=bf16
+ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget.
+MEGATRON_EXTRA_ARGS=''
+
+# Three complete real/pseudo/refiner passes on every update and during evaluation.
+# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2.
+LOOKAHEAD_MODE=parallel
+LOOKAHEAD_NUM_PASSES=3
+LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1
+LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto
+LOOKAHEAD_CHECKPOINT_REFINER=true
+LOOKAHEAD_REFINER_FFN_SIZE=5632
+LOOKAHEAD_REFINER_GATE_BIAS=-2
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2
+WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2
+
+# Frequent logs and a short sampled evaluation for the initial training check.
+WANDB_ENTITY=LoopTF-4-CSPs
+WANDB_PROJECT=loopTF-workspace
+WANDB_MODE=online
+WANDB_RESUME=never
+SAVE_INTERVAL=4000
+EVAL_AT_START=true
+EVAL_INTERVAL=100
+EVAL_ITERS=2
+LOG_INTERVAL=1
+CKPT_FORMAT=torch
+SEED=42
+
+# Pinned R3 single-node objective-strength experiment. No architecture changes.
+LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0
+SEED=43
+NNODES=1
+GPUS_PER_NODE=8
+TP=1
+PP=1
+MICRO_BATCH_SIZE=4
+GLOBAL_BATCH_SIZE=128
+TRAIN_ITERS=26703
+EXIT_INTERVAL=16000
+SAVE_INTERVAL=4000
+SAVE_RETAIN_INTERVAL=4000
+ENABLE_RECOMPUTE=false
+LOOKAHEAD_CHECKPOINT_REFINER=false
+EVAL_AT_START=true
+FINAL_EVAL_ITERS=270
+LOG_INTERVAL=1
+RESUME_POLICY=fresh
+REQUIRE_CHECKPOINT=false
+WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300
+WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/manifest.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    {
+      "path": "README.md",
+      "sha256": "3ed67538dda6cad8421469dc3b1c69a82e79677c02a41b07ae48b28d04ecf876",
+      "size": 2337
+    },
+    {
+      "path": "config.json",
+      "sha256": "73e86c7512e69d623c01aa788317b26a0c8da4c3a40cb6b6e62dd580e18238fe",
+      "size": 10067
+    },
+    {
+      "path": "launch_config.env",
+      "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26",
+      "size": 3907
+    },
+    {
+      "path": "model.safetensors",
+      "sha256": "db283fc16f1512d9fa2fe9037dc2fc1e2ac821580d426fe452e04c3f0b3eea65",
+      "size": 1322796152
+    },
+    {
+      "path": "training_recipe.json",
+      "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6",
+      "size": 4352
+    }
+  ],
+  "format": "safetensors_native_megatron_state_dict",
+  "iteration": 26703,
+  "note": "This manifest does not hash itself.",
+  "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109"
+}

Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db283fc16f1512d9fa2fe9037dc2fc1e2ac821580d426fe452e04c3f0b3eea65
+size 1322796152