diff --git a/LOOKAHEAD_NAME_MAP.json b/LOOKAHEAD_NAME_MAP.json new file mode 100644 index 0000000000000000000000000000000000000000..6b2436c57c0132bcb56914b26326ea0f99d12b7c --- /dev/null +++ b/LOOKAHEAD_NAME_MAP.json @@ -0,0 +1,429 @@ +{ + "schema_version": 1, + "pattern": "Lookahead-K3-LA2-Aa-Rr-Dd-Pp-Ff-seedN[-distinguishers]--wandb_id", + "decimal_separator": "p", + "legacy_defaults": "Missing legacy A/R/D/P/F options use the verified backward-compatible A0/R0/0/0/0 policy defaults. Seeds always come from saved W&B configuration.", + "runs": [ + { + "wandb_id": "my34axon", + "previous_wandb_name": "r4_la_hf_warmstart_s12000_to26703", + "training_run_name": "r4_la_hf_warmstart_s12000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s12000_to26703" + ] + }, + { + "wandb_id": "wkv2n5ub", + "previous_wandb_name": "r4_la_hf_warmstart_s20000_to26703", + "training_run_name": "r4_la_hf_warmstart_s20000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s20000_to26703" + ] + }, + { + "wandb_id": "7t0tefok", + "previous_wandb_name": "r4_la_hf_warmstart_s16000_to26703", + "training_run_name": "r4_la_hf_warmstart_s16000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s16000_to26703" + ] + }, + { + "wandb_id": "xhm12l4y", + "previous_wandb_name": "r3-a1-r0-d0-p0-f0-s42-7b--xhm12l4y", + "training_run_name": "r3_D0-P0-A1-R0_7b_seed42_20260920-023045", + "canonical_name": "Lookahead-K3-LA2-A1-R0-D0-P0-F0-seed42-7b--xhm12l4y", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A1", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "rsk51lmo", + "previous_wandb_name": "r3-a0-r1-d0-p0-f1-s42-7b--rsk51lmo", + "training_run_name": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "canonical_name": "Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R1", + "D": 0, + "P": 0, + "F": 1, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" + ] + }, + { + "wandb_id": "fob3gqiq", + "previous_wandb_name": "r3-a2-r1-d0-p0-f1-s42-7b--fob3gqiq", + "training_run_name": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "canonical_name": "Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R1", + "D": 0, + "P": 0, + "F": 1, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" + ] + }, + { + "wandb_id": "1jtjk6d7", + "previous_wandb_name": "r3-a2-r0-d0-p0-f0-s42-7b--1jtjk6d7", + "training_run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A2-R0_7b_seed42_20260919-160042" + ] + }, + { + "wandb_id": "gcwdjm01", + "previous_wandb_name": "r3-a2-r0-d0-p0p2-f0-s42-7b--gcwdjm01", + "training_run_name": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-A2-R0_7b_seed42_20260919-040210" + ] + }, + { + "wandb_id": "3pjop6r1", + "previous_wandb_name": "r3-a0-r0-d0p2-p0p2-f0-s42-7b--3pjop6r1", + "training_run_name": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0.2, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D1-A0-R0_7b_seed42_20260919-040211" + ] + }, + { + "wandb_id": "djlht18n", + "previous_wandb_name": "r3-a2-r0-d0p2-p0p2-f0-s42-7b--djlht18n", + "training_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0.2, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_d1_a2_r0_7b_seed42_20260919-035203-resume" + ] + }, + { + "wandb_id": "piob4h78", + "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-gstatesonly-s42-16kstep--piob4h78", + "training_run_name": "r3_lookahead_states_only_product_20260918-212932", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-grad-states-only-16kstep--piob4h78", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.05, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "b5xq9efg", + "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-gheadonly-s42-16kstep--b5xq9efg", + "training_run_name": "r3_lookahead_head_only_product_20260918-212932", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-grad-head-only-16kstep--b5xq9efg", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.05, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "7tv5vb6g", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-14b--7tv5vb6g", + "training_run_name": "20260918-LA14B-prod-02", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-14b--7tv5vb6g", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "3qdhqjxj", + "previous_wandb_name": "r3-a0-r0-competitive-d0-p0-f0-s42-16kstep--3qdhqjxj", + "training_run_name": "20260918-194754_competitive-16gpu-seed42-product", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-fusion-competitive-16kstep--3qdhqjxj", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "db7elm9j", + "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-s43-16kstep--db7elm9j", + "training_run_name": "r3_b300_alpha0p05-seed43_product_20260918-170108", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed43-16kstep--db7elm9j", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.05, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 43, + "hf_old_paths": [] + }, + { + "wandb_id": "bapl3lfs", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s43-7b--bapl3lfs", + "training_run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 43, + "hf_old_paths": [ + "r3_b300_alpha0p0-seed43_product_20260918-170109" + ] + }, + { + "wandb_id": "fo94dla9", + "previous_wandb_name": "r3-a0-r0-d0-p0p2-f0-s42-16kstep--fo94dla9", + "training_run_name": "r3_b300_alpha0p2-seed42_product_20260918-170100", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p2-F0-seed42-16kstep--fo94dla9", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "14vjphny", + "previous_wandb_name": "r3-a0-r0-d0-p0p1-f0-s42-16kstep--14vjphny", + "training_run_name": "r3_b300_alpha0p1-seed42_product_20260918-165348", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p1-F0-seed42-16kstep--14vjphny", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.1, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "83uufov3", + "previous_wandb_name": "r3-a0-r0-d0-p0p05-f0-s42-16kstep--83uufov3", + "training_run_name": "r3_lookahead_future05_16k_20260918-112504", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0p05-F0-seed42-16kstep--83uufov3", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0.05, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "cy5gslq2", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-16kstep--cy5gslq2", + "training_run_name": "r3_lookahead_control_16k_20260918-112311", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-16kstep--cy5gslq2", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [] + }, + { + "wandb_id": "9i6byar5", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-7b--9i6byar5", + "training_run_name": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928" + ] + } + ] +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/NAMING.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..3bc260b7aae4706a33dc13081de515770e70bd26 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "9i6byar5", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s42-7b--9i6byar5", + "training_run_name": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928" + ] +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0da95d8cb892fc21696fe962ce560145e69f4bdd --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/README.md @@ -0,0 +1,44 @@ +# Lookahead K3 LA2 — step 26703 + +Training run: `paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928`. Final checkpoint after **7,000,031,232 input tokens**; training exited successfully on 2026-09-18 (Asia/Shanghai). + +Native loopTF Lookahead weights, preserving tensor names and dtypes from the TP=1/PP=1 Megatron checkpoint. There are **661,315,072 parameters** in the complete model, including the Lookahead components. K3 uses three complete passes, each with one real and one pseudo backbone forward. LA2 is the existing design's naming label, not two extra pseudo steps. + +`model.safetensors`, `config.json`, `launch_config.env`, and `training_recipe.json` follow this repository's vanilla export convention. `config.json` also records constructor arguments and final validation; `manifest.json` records payload sizes and SHA256 hashes. This is a model-weights export, not a complete training-resume checkpoint. Optimizer, scheduler, RNG, dataloader state and credentials are excluded. + +## Load with the project implementation + +Use [loopTF-workspace](https://github.com/FFishy-git/loopTF-workspace) at training revision `432ef3046b5cf834ad4f61e3985cd3013343eeba` with its `src` directory on `PYTHONPATH` and the dependencies installed. This custom architecture is **not** a Transformers `AutoModel.from_pretrained` export. + +After downloading this step folder, set `export_dir` to its local path: + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig + +export_dir = Path("/path/to/step_26703") +config = json.loads((export_dir / "config.json").read_text()) +with torch.device("meta"): + model = LookaheadTransformer( + ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"]), + ) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +assert all(key.startswith("model.") for key in state) +model.load_state_dict( + {key.removeprefix("model."): value for key, value in state.items()}, + strict=True, assign=True, +) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` described in `config.json` (32,001 active tokens). The vocabulary tensors retain padding to 32,064. The launch snapshot records base settings; `training_run` in `config.json` is the effective timestamped run identity. + +Final held-out pass-3 CE: **2.27749586**, perplexity **9.75222887**, over 34,560 sequences / 70,778,880 input tokens. No new evaluation or training was performed during export. + +[Weights & Biases run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/9i6byar5) diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/config.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..971f86dcf1487878ab218f3f463a4060c5d903fb --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/config.json @@ -0,0 +1,140 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 26703, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "crop_padding_before_loss": true, + "dropout": 0.0, + "eval_passes": 3, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": { + "eval/checkpoint_step": 26703, + "eval/final_val/dataset_id": "config-a71f5512f13ffd6f", + "eval/final_val/final_ce": 2.277495861053467, + "eval/final_val/final_ppl": 9.752228871149752, + "eval/final_val/num_input_tokens": 70778880, + "eval/final_val/num_sequences": 34560, + "eval/final_val/num_tokens": 70778880, + "eval/final_val/pass_1_ce": 2.3000683784484863, + "eval/final_val/pass_1_ppl": 9.974864497254137, + "eval/final_val/pass_2_ce": 2.2785534858703613, + "eval/final_val/pass_2_ppl": 9.762548526623844, + "eval/final_val/pass_3_ce": 2.277495861053467, + "eval/final_val/pass_3_ppl": 9.752228871149752, + "eval/final_val/pass_count": 3, + "eval/is_final": true + }, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "loading_note": "Native LookaheadMegatronModel state_dict keys, not a Transformers AutoModel export. For LookaheadTransformer, remove exactly the leading model. prefix and load with strict=True; assign=True preserves saved dtypes. No optimizer, scheduler, RNG, or data-loader state is included.", + "logit_softcap": 15.0, + "lookahead_config": { + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "feedback_loss_weight": 1.0, + "mode": "parallel", + "num_passes": 3, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632 + }, + "lookahead_mode": "parallel", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "naming_note": "K3 is three complete passes including the seed. LA2 labels the existing one-real-plus-one-pseudo-forward design per pass; it does not add two future pseudo steps.", + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "train_passes": 3, + "training_code_revision": "432ef3046b5cf834ad4f61e3985cd3013343eeba", + "training_run": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928", + "training_tokens": 7000031232, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..08ec487fa8dd9a9ec3d713c23c225e9e974dfa82 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/launch_config.env @@ -0,0 +1,100 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=false # Profiled on 32 B300 GPUs; avoids recompute with ~101.4 GiB peak allocation/GPU. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=false +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..874f1370af69cabc272568ad2f8d603e914793ec --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 26703, + "files": { + "README.md": { + "sha256": "3b876c301ac9b01783e3cae3a5c0247e496f73175d7302de1e3566beb90b9d58", + "size": 2744 + }, + "config.json": { + "sha256": "64c12b7b17ce62c57bf71f69c6223fad2f4ad33d190fbe67976b333b3c51faab", + "size": 4984 + }, + "launch_config.env": { + "sha256": "4939b5b392c71bc4111cb30f954903da4b5296160589603c8fe9aaae3b988d59", + "size": 3365 + }, + "model.safetensors": { + "sha256": "47c9c162f943bde560d66e39809b2e70f27e7c72f5bb117ec96194176d489c83", + "size": 1322796200 + }, + "training_recipe.json": { + "sha256": "7688bd955bc0d6fb1b134a5f92ab88d000a87a67f0cd051f079445b2e3b74130", + "size": 4092 + } + }, + "manifest_note": "Hashes cover the other payload files; this manifest does not hash itself.", + "path_in_repo": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928/step_26703", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "paper2604_lookahead_476m_7b_online_K3_LA2_20260917-190928" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8620377555ac1af6ab1149e8716806a3081547b1 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c9c162f943bde560d66e39809b2e70f27e7c72f5bb117ec96194176d489c83 +size 1322796200 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..564564908f86723fe87e748bdbb3031ca97360d9 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed42-7b--9i6byar5/step_26703/training_recipe.json @@ -0,0 +1,142 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999 + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/NAMING.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..c89b76b60c4031e21cbae4edc39f4950a3f9c914 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "bapl3lfs", + "previous_wandb_name": "r3-a0-r0-d0-p0-f0-s43-7b--bapl3lfs", + "training_run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 43, + "hf_old_paths": [ + "r3_b300_alpha0p0-seed43_product_20260918-170109" + ] +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5689f8bf63a25a020bfde817847745c9af833dc6 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/README.md @@ -0,0 +1,7 @@ +# r3_b300_alpha0p0-seed43_product_20260918-170109: native resumable checkpoint at step12000 + +Private training-state backup, not a Transformers model export. Training was cancelled by direct user approval. Last logged completed update was13933;1933 subsequent-to-checkpoint logged updates are unsaved. This bundle covers step12000 only, not later logged progress. + +Preserves native model/optimizer/scheduler/RNG and eight DP-rank loader states. Restore with matching source7e264fdbf158f9a8f41cb6d09b29ed71b867ca27 plus code.diff and source_overlay, Megatron d046b0de593b1802c78a7664765174c031eb452c, same TP1/PP1/DP8 and pinned tokenizer/data. Source checkout access is required; dataset is excluded. Review launch_config.env and resume_provenance.json. Point LOAD_PATH to the restored checkpoints/ directory, DATALOADER_SAVE to the restored dataloader/ directory, RESUME_POLICY=auto, REQUIRE_CHECKPOINT=true, WANDB_RUN_ID=bapl3lfs, WANDB_RESUME=must and preserve product/descriptive tags plus resume. Keep original26703 LR horizon and16000 total-step cap. Paths in the frozen config must be relocated explicitly as needed. No resume launch is authorized by this backup. + +No execution test of restoration performed. Checksum manifest excludes itself; immutable remote verification is pending. Never load untrusted native pickle files. diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d7209ef79b510d2b65a0177b2ca785cd23c18e6 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ba9446ef7ad5452940570bdf51ed3d0668c6f60476a459b1fd3215229c4b2ae +size 6974760755 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/latest_checkpointed_iteration.txt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e334181b4006249f910c0a5a73a7d6cd17894f90 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/checkpoints/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +12000 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/code.diff b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/code.diff new file mode 100644 index 0000000000000000000000000000000000000000..0778db4e00056ca3ed2b8f95b9ad1bf76b2e57ad --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/code.diff @@ -0,0 +1,14 @@ +diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py +index fc03aaf..8047ea2 100644 +--- a/src/loop_transformer_backends/megatron/runtime.py ++++ b/src/loop_transformer_backends/megatron/runtime.py +@@ -175,6 +175,9 @@ def run( + ) + model_config = architecture.build_model_config(args) + with ExitStack() as hooks: ++ if writer is not None: ++ from .wandb_storage import install_wandb_file_hashing ++ hooks.callback(install_wandb_file_hashing()) + hooks.callback(install_ft_args_hook(args)) + if args.training_recipe == DEFAULT_RECIPE: + hooks.enter_context(paper_checkpoint_context()) diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt new file mode 100644 index 0000000000000000000000000000000000000000..62ab3260511683155b62862be85704f13158bb8b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2de254e0312563219685177629aaf79a70aad8ebf933963916c025a19cbffcd3 +size 6665781 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0e65724a97af1d50bfa37b8fcc2e03fee89adfd --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7039b2866ebc92a8ff3cc124ad1ea212565909d6135c84f70882164e1293bea3 +size 6762549 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt new file mode 100644 index 0000000000000000000000000000000000000000..5974ae500199d0c77fe8855560e6ba45f2bf8796 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92f0fac9c5ee1f888ca73800dd487f6bd5f5e5cf61527f17aff72c83978592a +size 6338485 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt new file mode 100644 index 0000000000000000000000000000000000000000..34c806805322a675437a6f94b123a8687473a1c2 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d668390740b3bc9067316c2c06e4854fab1e8b9a8808ef49603abcbb552606 +size 6927349 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt new file mode 100644 index 0000000000000000000000000000000000000000..820d5b2bd7dc5204c9e1f000960a185ca7526945 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15715a11d4e1dade60eed8965fc7061d0492330e4b6a2b1b1f04ee61dae44c6d +size 6409205 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt new file mode 100644 index 0000000000000000000000000000000000000000..fce7f50bc6b250fa434254b4e8b94450bd6fb695 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfa876b9239b76f5aa13c77f85b1183627337d058f8b13bc2a90e9f7478ac65c +size 6878517 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt new file mode 100644 index 0000000000000000000000000000000000000000..342adf86c861a401c53e331259a5c2fe9e5ea5d9 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55d4dad7cc4b3481f750dcb4d4ba6f1c7ee03fe13628454efc15b274e854b5f3 +size 6970293 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3a05fa061b15aa01cae1ad5500f6a5032ab85e8 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2346d6fe631dadf7b6694c9863a182214bafaa6527c3dd01dd620aff0a7e8783 +size 6647861 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..4b7323d93105beeafde639fdd5209f2d2c1d0674 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/launch_config.env @@ -0,0 +1,123 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=true +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 + +# Pinned R3 single-node objective-strength experiment. No architecture changes. +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0 +SEED=43 +NNODES=1 +GPUS_PER_NODE=8 +TP=1 +PP=1 +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +TRAIN_ITERS=26703 +EXIT_INTERVAL=16000 +SAVE_INTERVAL=4000 +SAVE_RETAIN_INTERVAL=4000 +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +EVAL_AT_START=true +FINAL_EVAL_ITERS=270 +LOG_INTERVAL=1 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false +WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300 +WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a25ff0f581ab167c8cdbd17c71aa1573367b9d28 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/manifest.json @@ -0,0 +1,77 @@ +{ + "repo_id": "Siyuc/loopTF-workspace", + "path_in_repo": "r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000", + "format": "native_megatron_full_training_resume", + "saved_step": 12000, + "files": { + "README.md": { + "bytes": 1334, + "sha256": "d2e6647b39d233d5086c1df9208336dc3c4fe0bd689af9dbef5051b3f877ad19" + }, + "checkpoints/iter_0012000/mp_rank_00/model_optim_rng.pt": { + "bytes": 6974760755, + "sha256": "1ba9446ef7ad5452940570bdf51ed3d0668c6f60476a459b1fd3215229c4b2ae" + }, + "checkpoints/latest_checkpointed_iteration.txt": { + "bytes": 6, + "sha256": "cc9f23a77ecea60b29786c934ac6c3f8fbe766405b3dc3f2b4b02679be56ba88" + }, + "code.diff": { + "bytes": 745, + "sha256": "09aec79ad20ab40065f952858ac9ce6cfe9f7590ae35646d28c13a9b7e1dad0c" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank000.pt": { + "bytes": 6665781, + "sha256": "2de254e0312563219685177629aaf79a70aad8ebf933963916c025a19cbffcd3" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank001.pt": { + "bytes": 6762549, + "sha256": "7039b2866ebc92a8ff3cc124ad1ea212565909d6135c84f70882164e1293bea3" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank002.pt": { + "bytes": 6338485, + "sha256": "c92f0fac9c5ee1f888ca73800dd487f6bd5f5e5cf61527f17aff72c83978592a" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank003.pt": { + "bytes": 6927349, + "sha256": "23d668390740b3bc9067316c2c06e4854fab1e8b9a8808ef49603abcbb552606" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank004.pt": { + "bytes": 6409205, + "sha256": "15715a11d4e1dade60eed8965fc7061d0492330e4b6a2b1b1f04ee61dae44c6d" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank005.pt": { + "bytes": 6878517, + "sha256": "bfa876b9239b76f5aa13c77f85b1183627337d058f8b13bc2a90e9f7478ac65c" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank006.pt": { + "bytes": 6970293, + "sha256": "55d4dad7cc4b3481f750dcb4d4ba6f1c7ee03fe13628454efc15b274e854b5f3" + }, + "dataloader/iter_0012000/mp_rank_00/train_dataloader_dprank007.pt": { + "bytes": 6647861, + "sha256": "2346d6fe631dadf7b6694c9863a182214bafaa6527c3dd01dd620aff0a7e8783" + }, + "launch_config.env": { + "bytes": 3907, + "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26" + }, + "project_revision.txt": { + "bytes": 41, + "sha256": "d9cf81883983241a3d6f60c54a46ce386c0c2db114ae17e8ad5fb67dc27795f2" + }, + "resume_provenance.json": { + "bytes": 2483, + "sha256": "c00d741cde09d95839669c88d50cc79c86876fd624608e7eb71cf29bcfa544cc" + }, + "source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py": { + "bytes": 672, + "sha256": "976b04f0d3a471acb4542167795f7d17ebc897c533b1f1ebc93c53e4cb01ffff" + }, + "training_recipe.json": { + "bytes": 4123, + "sha256": "7a62b1c7ab13f0b5d7d3b05de1eab2b338813322a885cb9f8564fe1c1e7189fd" + } + }, + "manifest_excludes_itself": true +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/project_revision.txt b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/project_revision.txt new file mode 100644 index 0000000000000000000000000000000000000000..26305f95475c33f169f17e263b7c39e3c0146428 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/project_revision.txt @@ -0,0 +1 @@ +7e264fdbf158f9a8f41cb6d09b29ed71b867ca27 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/resume_provenance.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/resume_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..81ea0a3da24f765f6c68ffed590ac340bc3832ac --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/resume_provenance.json @@ -0,0 +1,94 @@ +{ + "format": "native_megatron_full_training_resume", + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "saved_step": 12000, + "cancelled": true, + "cancellation_status": "user_cancelled_verified", + "last_logged_step_at_preparation": 13812, + "source_revision": "7e264fdbf158f9a8f41cb6d09b29ed71b867ca27", + "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c", + "alpha": 0.0, + "seed": 43, + "tp": 1, + "pp": 1, + "dp": 8, + "components": { + "model": true, + "optimizer": true, + "opt_param_scheduler": true, + "rng_state": true + }, + "rng_state_count": 1, + "loader_states": [ + { + "rank": 0, + "bytes": 6665781, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 1, + "bytes": 6762549, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 2, + "bytes": 6338485, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 3, + "bytes": 6927349, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 4, + "bytes": 6409205, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 5, + "bytes": 6878517, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 6, + "bytes": 6970293, + "top_level_keys": [ + "dataloader_state_dict" + ] + }, + { + "rank": 7, + "bytes": 6647861, + "top_level_keys": [ + "dataloader_state_dict" + ] + } + ], + "wandb_id": "bapl3lfs", + "lr_horizon": 26703, + "total_executed_step_cap": 16000, + "missing_state": [], + "limitations": [ + "Native trusted-pickle format; load only trusted checkpoints with pinned source/dependencies.", + "No training-resume execution performed.", + "Only saved-step state is backed up; later optimizer updates are not part of this payload.", + "Source code repository access and pinned dataset/tokenizer are required; training data is excluded." + ], + "rng_state_note": "One shared native RNG record is expected: data_parallel_random_init=False. Pinned Megatron checkpointing.py saves one record and restores index0 for each DP rank. All eight distinct data-loader states are included.", + "stop_step": 13933, + "stop_step_definition": "Last fully logged completed optimizer update; an interrupted in-flight update is not claimed.", + "unsaved_logged_updates": 1933 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..4dbaed49a2447baddc01f7373e1edc43748b6a11 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/source_overlay/src/loop_transformer_backends/megatron/wandb_storage.py @@ -0,0 +1,18 @@ +"""Buffered W&B file hashing for shared checkpoint filesystems.""" +from __future__ import annotations + +def _buffered_file_hasher(hasher, *paths): + for path in sorted(map(str, paths)): + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(128 * 1024), b""): + hasher.update(chunk) + return hasher + +def install_wandb_file_hashing(): + """Avoid fatal mmap faults; retain MD5/xxHash and sorted-path contracts.""" + from wandb.sdk.lib import hashutil + original = hashutil._file_hasher + hashutil._file_hasher = _buffered_file_hasher + def restore(): + hashutil._file_hasher = original + return restore diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..e3ad0e87009331cf8ced7da73ab76590e5d49ea4 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_12000/training_recipe.json @@ -0,0 +1,143 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0 + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4de3ac6b589edee726e4f0a9dfb22fe9d91d506f --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/README.md @@ -0,0 +1,40 @@ +# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 16000 + +Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment. +Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint. +This checkpoint consumed 4194304000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable. +The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle. + +Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769 +Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132. +W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs +Access to the code repository is required. Paper backbone source revision is separately recorded in config.json. + +## Loading + +Use the pinned repository and its dependencies, with its src directory on PYTHONPATH. +These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig + +cfg = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**cfg["model_config"]), + LookaheadConfig(**cfg["lookahead_config"])) +state = load_file("model.safetensors") +state.update(cfg["non_tensor_state"]) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k[len("model."):]: v for k, v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass. +The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments. +Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export. +No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700. diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/config.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6b7cf875a973128b3a846c87bc874fc6bbf5386f --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/config.json @@ -0,0 +1,237 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "data_parallel_size": 8, + "effective_launch_settings": { + "ADAM_BETA1": "0.8", + "ADAM_BETA2": "0.95", + "ADAM_EPS": "1e-10", + "ATTENTION_DROPOUT": "0", + "ATTENTION_OUTPUT_GATE": "false", + "AUX_MTP_HORIZON": "1", + "AUX_MTP_LOSS_WEIGHT": "0.3", + "AUX_MTP_RANK": "128", + "BACKBONE_PROFILE": "paper2604_vanilla", + "CKPT_FORMAT": "torch", + "CLIP_GRAD": "0.0", + "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader", + "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_PREFLIGHT": "false", + "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data", + "DISABLE_BIAS_LINEAR": "true", + "DISABLE_WANDB": "false", + "ENABLE_RECOMPUTE": "false", + "EVAL_AT_START": "true", + "EVAL_INTERVAL": "100", + "EVAL_ITERS": "2", + "EVAL_ONLY": "false", + "EXIT_INTERVAL": "26703", + "EXIT_SIGNAL_HANDLER": "false", + "EXPECTED_INPUT_LENGTH": "2048", + "EXPECTED_RECORD_LENGTH": "2049", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "FFN_HIDDEN_SIZE": "5632", + "FINAL_EVAL_ITERS": "270", + "GLOBAL_BATCH_SIZE": "128", + "GPUS_PER_NODE": "8", + "GROUP_QUERY_ATTENTION": "false", + "HIDDEN_DROPOUT": "0", + "HIDDEN_SIZE": "1408", + "LAYERNORM_EPSILON": "0.000001", + "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints", + "LOG_INTERVAL": "1", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LR": "0.014", + "LR_DECAY_STYLE": "linear", + "LR_WSD_DECAY_STYLE": "linear", + "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64", + "MAX_POSITION_EMBEDDINGS": "2048", + "MEGATRON_EXTRA_ARGS": "", + "MICRO_BATCH_SIZE": "4", + "MIN_LR": "0.0007470638533717787", + "MIN_MODEL_INPUT_TOKENS": "7000000000", + "MUON_MOMENTUM": "0.95", + "MUON_NESTEROV": "true", + "NORMALIZATION": "RMSNorm", + "NUM_ATTENTION_HEADS": "11", + "NUM_LAYERS": "20", + "NUM_QUERY_GROUPS": "11", + "NUM_WORKERS": "0", + "ONLINE_DATA": "true", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "ONLINE_LATENCY_LOG_DIR": "", + "ONLINE_PACK_SEQUENCES": "false", + "ONLINE_PREFETCH_BATCHES": "16", + "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", + "ONLINE_SHUFFLE_BUFFER_RECORDS": "128", + "ONLINE_SOURCE_MODE": "local", + "ONLINE_SOURCE_RETRY_LIMIT": "8", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TEXT_COLUMN": "text", + "ONLINE_TOKENIZER_BATCH_SIZE": "256", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "ONLINE_VALIDATION_MANIFEST": "", + "OPTIMIZER": "muonh", + "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output", + "PAPER_EMBEDDING_LR": "0.3", + "PAPER_NORM_LR": "0.005", + "PAPER_OTHER_LR": "0.005", + "PAPER_REFERENCE_BATCH_TOKENS": "262144", + "PAPER_REFERENCE_TOKENS": "983315200", + "PAPER_REFERENCE_WIDTH": "640", + "PAPER_UNEMBEDDING_LR": "0.004", + "POSITION_EMBEDDING_TYPE": "rope", + "PP": "1", + "PRECISION": "bf16", + "QK_LAYERNORM": "false", + "RECIPE_OVERRIDES": "{}", + "RECIPE_REQUESTED_BATCH_TOKENS": "262144", + "RECOMPUTE_GRANULARITY": "full", + "RECOMPUTE_METHOD": "uniform", + "RECOMPUTE_NUM_LAYERS": "1", + "REQUIRE_CHECKPOINT": "true", + "RESUME_POLICY": "auto", + "ROTARY_BASE": "10000", + "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847", + "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "SAVE_INTERVAL": "4000", + "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints", + "SAVE_RETAIN_INTERVAL": "4000", + "SEED": "43", + "SEQ_LENGTH": "2048", + "SWIGLU": "false", + "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard", + "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "TOKENIZER_TYPE": "HuggingFaceTokenizer", + "TP": "1", + "TRAINING_RECIPE": "paper2604_21106", + "TRAIN_ARCHITECTURE": "lookahead", + "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document", + "TRAIN_ITERS": "26703", + "UNTIE_EMBEDDINGS": "true", + "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "VOCAB_SIZE": "32001", + "WANDB_ENTITY": "LoopTF-4-CSPs", + "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "WANDB_MODE": "online", + "WANDB_PROJECT": "loopTF-workspace", + "WANDB_RESUME": "must", + "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918", + "WANDB_RUN_ID": "bapl3lfs", + "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb", + "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200", + "WARMUP_ITERS": "0", + "WEIGHT_DECAY": "0.0", + "WINDOW_ATTN_SKIP_FREQ": "1", + "WINDOW_SIZE": "none", + "WSD_DECAY_ITERS": "0" + }, + "iteration": 16000, + "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.", + "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.", + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "non_tensor_state": {}, + "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 661315072, + "trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "tensor_parallel_size": 1, + "tokenizer": { + "active_vocab_size": 32001, + "added_tokens": [ + "<|assistant_end|>" + ], + "padded_vocab_size": 32064, + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training": { + "completed_run": true, + "consumed_input_tokens": 4194304000, + "consumed_train_samples": 2048000, + "final_checkpoint": false, + "global_batch_size": 128, + "micro_batch_size": 4, + "planned_input_tokens": 7000031232, + "resumed_from_step": 12000, + "seed": 43, + "sequence_length": 2048, + "train_iters": 26703 + }, + "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace", + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..4b7323d93105beeafde639fdd5209f2d2c1d0674 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/launch_config.env @@ -0,0 +1,123 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=true +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 + +# Pinned R3 single-node objective-strength experiment. No architecture changes. +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0 +SEED=43 +NNODES=1 +GPUS_PER_NODE=8 +TP=1 +PP=1 +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +TRAIN_ITERS=26703 +EXIT_INTERVAL=16000 +SAVE_INTERVAL=4000 +SAVE_RETAIN_INTERVAL=4000 +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +EVAL_AT_START=true +FINAL_EVAL_ITERS=270 +LOG_INTERVAL=1 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false +WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300 +WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..f4d114ad79a09989899a38af9b20ff9beb1954af --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/manifest.json @@ -0,0 +1,33 @@ +{ + "files": [ + { + "path": "README.md", + "sha256": "2759094b2b3e6649ec56b17b2674144884f6cba9a2114c0997539c729daa6e48", + "size": 2352 + }, + { + "path": "config.json", + "sha256": "15826e3ae8ecfaa74cc08baeaf202aecfef75ce20d1f60db11d9b51a235c319b", + "size": 10068 + }, + { + "path": "launch_config.env", + "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26", + "size": 3907 + }, + { + "path": "model.safetensors", + "sha256": "4d66347d78097750130c50e659f02a134eed43c90fd1e6d8779c8788795998e9", + "size": 1322796152 + }, + { + "path": "training_recipe.json", + "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6", + "size": 4352 + } + ], + "format": "safetensors_native_megatron_state_dict", + "iteration": 16000, + "note": "This manifest does not hash itself.", + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a9e6f66722c446c413f43b6ef89facefc1f0d13 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d66347d78097750130c50e659f02a134eed43c90fd1e6d8779c8788795998e9 +size 1322796152 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..4452defdeefcb469f5d6ed08baab96f45374163b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_16000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6ca0c30c6839571ec4b422117cab0bb224f6fae7 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/README.md @@ -0,0 +1,40 @@ +# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 20000 + +Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment. +Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint. +This checkpoint consumed 5242880000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable. +The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle. + +Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769 +Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132. +W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs +Access to the code repository is required. Paper backbone source revision is separately recorded in config.json. + +## Loading + +Use the pinned repository and its dependencies, with its src directory on PYTHONPATH. +These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig + +cfg = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**cfg["model_config"]), + LookaheadConfig(**cfg["lookahead_config"])) +state = load_file("model.safetensors") +state.update(cfg["non_tensor_state"]) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k[len("model."):]: v for k, v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass. +The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments. +Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export. +No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700. diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/config.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5710e536594357b5d8b920f8f6814b942339ea34 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/config.json @@ -0,0 +1,237 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "data_parallel_size": 8, + "effective_launch_settings": { + "ADAM_BETA1": "0.8", + "ADAM_BETA2": "0.95", + "ADAM_EPS": "1e-10", + "ATTENTION_DROPOUT": "0", + "ATTENTION_OUTPUT_GATE": "false", + "AUX_MTP_HORIZON": "1", + "AUX_MTP_LOSS_WEIGHT": "0.3", + "AUX_MTP_RANK": "128", + "BACKBONE_PROFILE": "paper2604_vanilla", + "CKPT_FORMAT": "torch", + "CLIP_GRAD": "0.0", + "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader", + "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_PREFLIGHT": "false", + "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data", + "DISABLE_BIAS_LINEAR": "true", + "DISABLE_WANDB": "false", + "ENABLE_RECOMPUTE": "false", + "EVAL_AT_START": "true", + "EVAL_INTERVAL": "100", + "EVAL_ITERS": "2", + "EVAL_ONLY": "false", + "EXIT_INTERVAL": "26703", + "EXIT_SIGNAL_HANDLER": "false", + "EXPECTED_INPUT_LENGTH": "2048", + "EXPECTED_RECORD_LENGTH": "2049", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "FFN_HIDDEN_SIZE": "5632", + "FINAL_EVAL_ITERS": "270", + "GLOBAL_BATCH_SIZE": "128", + "GPUS_PER_NODE": "8", + "GROUP_QUERY_ATTENTION": "false", + "HIDDEN_DROPOUT": "0", + "HIDDEN_SIZE": "1408", + "LAYERNORM_EPSILON": "0.000001", + "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints", + "LOG_INTERVAL": "1", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LR": "0.014", + "LR_DECAY_STYLE": "linear", + "LR_WSD_DECAY_STYLE": "linear", + "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64", + "MAX_POSITION_EMBEDDINGS": "2048", + "MEGATRON_EXTRA_ARGS": "", + "MICRO_BATCH_SIZE": "4", + "MIN_LR": "0.0007470638533717787", + "MIN_MODEL_INPUT_TOKENS": "7000000000", + "MUON_MOMENTUM": "0.95", + "MUON_NESTEROV": "true", + "NORMALIZATION": "RMSNorm", + "NUM_ATTENTION_HEADS": "11", + "NUM_LAYERS": "20", + "NUM_QUERY_GROUPS": "11", + "NUM_WORKERS": "0", + "ONLINE_DATA": "true", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "ONLINE_LATENCY_LOG_DIR": "", + "ONLINE_PACK_SEQUENCES": "false", + "ONLINE_PREFETCH_BATCHES": "16", + "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", + "ONLINE_SHUFFLE_BUFFER_RECORDS": "128", + "ONLINE_SOURCE_MODE": "local", + "ONLINE_SOURCE_RETRY_LIMIT": "8", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TEXT_COLUMN": "text", + "ONLINE_TOKENIZER_BATCH_SIZE": "256", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "ONLINE_VALIDATION_MANIFEST": "", + "OPTIMIZER": "muonh", + "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output", + "PAPER_EMBEDDING_LR": "0.3", + "PAPER_NORM_LR": "0.005", + "PAPER_OTHER_LR": "0.005", + "PAPER_REFERENCE_BATCH_TOKENS": "262144", + "PAPER_REFERENCE_TOKENS": "983315200", + "PAPER_REFERENCE_WIDTH": "640", + "PAPER_UNEMBEDDING_LR": "0.004", + "POSITION_EMBEDDING_TYPE": "rope", + "PP": "1", + "PRECISION": "bf16", + "QK_LAYERNORM": "false", + "RECIPE_OVERRIDES": "{}", + "RECIPE_REQUESTED_BATCH_TOKENS": "262144", + "RECOMPUTE_GRANULARITY": "full", + "RECOMPUTE_METHOD": "uniform", + "RECOMPUTE_NUM_LAYERS": "1", + "REQUIRE_CHECKPOINT": "true", + "RESUME_POLICY": "auto", + "ROTARY_BASE": "10000", + "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847", + "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "SAVE_INTERVAL": "4000", + "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints", + "SAVE_RETAIN_INTERVAL": "4000", + "SEED": "43", + "SEQ_LENGTH": "2048", + "SWIGLU": "false", + "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard", + "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "TOKENIZER_TYPE": "HuggingFaceTokenizer", + "TP": "1", + "TRAINING_RECIPE": "paper2604_21106", + "TRAIN_ARCHITECTURE": "lookahead", + "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document", + "TRAIN_ITERS": "26703", + "UNTIE_EMBEDDINGS": "true", + "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "VOCAB_SIZE": "32001", + "WANDB_ENTITY": "LoopTF-4-CSPs", + "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "WANDB_MODE": "online", + "WANDB_PROJECT": "loopTF-workspace", + "WANDB_RESUME": "must", + "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918", + "WANDB_RUN_ID": "bapl3lfs", + "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb", + "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200", + "WARMUP_ITERS": "0", + "WEIGHT_DECAY": "0.0", + "WINDOW_ATTN_SKIP_FREQ": "1", + "WINDOW_SIZE": "none", + "WSD_DECAY_ITERS": "0" + }, + "iteration": 20000, + "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.", + "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.", + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "non_tensor_state": {}, + "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 661315072, + "trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "tensor_parallel_size": 1, + "tokenizer": { + "active_vocab_size": 32001, + "added_tokens": [ + "<|assistant_end|>" + ], + "padded_vocab_size": 32064, + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training": { + "completed_run": true, + "consumed_input_tokens": 5242880000, + "consumed_train_samples": 2560000, + "final_checkpoint": false, + "global_batch_size": 128, + "micro_batch_size": 4, + "planned_input_tokens": 7000031232, + "resumed_from_step": 12000, + "seed": 43, + "sequence_length": 2048, + "train_iters": 26703 + }, + "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace", + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..4b7323d93105beeafde639fdd5209f2d2c1d0674 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/launch_config.env @@ -0,0 +1,123 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=true +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 + +# Pinned R3 single-node objective-strength experiment. No architecture changes. +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0 +SEED=43 +NNODES=1 +GPUS_PER_NODE=8 +TP=1 +PP=1 +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +TRAIN_ITERS=26703 +EXIT_INTERVAL=16000 +SAVE_INTERVAL=4000 +SAVE_RETAIN_INTERVAL=4000 +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +EVAL_AT_START=true +FINAL_EVAL_ITERS=270 +LOG_INTERVAL=1 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false +WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300 +WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..07492211edfa15dca88aede998a44c1217107b4e --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/manifest.json @@ -0,0 +1,33 @@ +{ + "files": [ + { + "path": "README.md", + "sha256": "306b3f2228014d8fabf82bcd4365ffefc3b43db8a2796f3c44d9f9997a9d9f28", + "size": 2352 + }, + { + "path": "config.json", + "sha256": "a6be34edaa2940c5ca6fca659333ffeb081347394208c841b0bc95f8f08b3e40", + "size": 10068 + }, + { + "path": "launch_config.env", + "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26", + "size": 3907 + }, + { + "path": "model.safetensors", + "sha256": "9a9789ee2c00ea0bfa583a11b86d8f4cb48bc7821a6cdf907048823acaee1176", + "size": 1322796152 + }, + { + "path": "training_recipe.json", + "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6", + "size": 4352 + } + ], + "format": "safetensors_native_megatron_state_dict", + "iteration": 20000, + "note": "This manifest does not hash itself.", + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c56c514e12aface31c2461e1c58f1fc1cf2f7ecd --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9789ee2c00ea0bfa583a11b86d8f4cb48bc7821a6cdf907048823acaee1176 +size 1322796152 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..4452defdeefcb469f5d6ed08baab96f45374163b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_20000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9f83764977e85724998e1a1dc0135a8afe9f4e44 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/README.md @@ -0,0 +1,40 @@ +# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 24000 + +Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment. +Training finished at update 26703 (7000031232 input tokens); this checkpoint is an intermediate retained checkpoint. +This checkpoint consumed 6291456000 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable. +The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle. + +Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769 +Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132. +W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs +Access to the code repository is required. Paper backbone source revision is separately recorded in config.json. + +## Loading + +Use the pinned repository and its dependencies, with its src directory on PYTHONPATH. +These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig + +cfg = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**cfg["model_config"]), + LookaheadConfig(**cfg["lookahead_config"])) +state = load_file("model.safetensors") +state.update(cfg["non_tensor_state"]) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k[len("model."):]: v for k, v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass. +The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments. +Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export. +No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700. diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/config.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cc6009df3827907ec7223a7b59026c240f4bbc95 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/config.json @@ -0,0 +1,237 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "data_parallel_size": 8, + "effective_launch_settings": { + "ADAM_BETA1": "0.8", + "ADAM_BETA2": "0.95", + "ADAM_EPS": "1e-10", + "ATTENTION_DROPOUT": "0", + "ATTENTION_OUTPUT_GATE": "false", + "AUX_MTP_HORIZON": "1", + "AUX_MTP_LOSS_WEIGHT": "0.3", + "AUX_MTP_RANK": "128", + "BACKBONE_PROFILE": "paper2604_vanilla", + "CKPT_FORMAT": "torch", + "CLIP_GRAD": "0.0", + "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader", + "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_PREFLIGHT": "false", + "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data", + "DISABLE_BIAS_LINEAR": "true", + "DISABLE_WANDB": "false", + "ENABLE_RECOMPUTE": "false", + "EVAL_AT_START": "true", + "EVAL_INTERVAL": "100", + "EVAL_ITERS": "2", + "EVAL_ONLY": "false", + "EXIT_INTERVAL": "26703", + "EXIT_SIGNAL_HANDLER": "false", + "EXPECTED_INPUT_LENGTH": "2048", + "EXPECTED_RECORD_LENGTH": "2049", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "FFN_HIDDEN_SIZE": "5632", + "FINAL_EVAL_ITERS": "270", + "GLOBAL_BATCH_SIZE": "128", + "GPUS_PER_NODE": "8", + "GROUP_QUERY_ATTENTION": "false", + "HIDDEN_DROPOUT": "0", + "HIDDEN_SIZE": "1408", + "LAYERNORM_EPSILON": "0.000001", + "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints", + "LOG_INTERVAL": "1", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LR": "0.014", + "LR_DECAY_STYLE": "linear", + "LR_WSD_DECAY_STYLE": "linear", + "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64", + "MAX_POSITION_EMBEDDINGS": "2048", + "MEGATRON_EXTRA_ARGS": "", + "MICRO_BATCH_SIZE": "4", + "MIN_LR": "0.0007470638533717787", + "MIN_MODEL_INPUT_TOKENS": "7000000000", + "MUON_MOMENTUM": "0.95", + "MUON_NESTEROV": "true", + "NORMALIZATION": "RMSNorm", + "NUM_ATTENTION_HEADS": "11", + "NUM_LAYERS": "20", + "NUM_QUERY_GROUPS": "11", + "NUM_WORKERS": "0", + "ONLINE_DATA": "true", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "ONLINE_LATENCY_LOG_DIR": "", + "ONLINE_PACK_SEQUENCES": "false", + "ONLINE_PREFETCH_BATCHES": "16", + "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", + "ONLINE_SHUFFLE_BUFFER_RECORDS": "128", + "ONLINE_SOURCE_MODE": "local", + "ONLINE_SOURCE_RETRY_LIMIT": "8", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TEXT_COLUMN": "text", + "ONLINE_TOKENIZER_BATCH_SIZE": "256", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "ONLINE_VALIDATION_MANIFEST": "", + "OPTIMIZER": "muonh", + "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output", + "PAPER_EMBEDDING_LR": "0.3", + "PAPER_NORM_LR": "0.005", + "PAPER_OTHER_LR": "0.005", + "PAPER_REFERENCE_BATCH_TOKENS": "262144", + "PAPER_REFERENCE_TOKENS": "983315200", + "PAPER_REFERENCE_WIDTH": "640", + "PAPER_UNEMBEDDING_LR": "0.004", + "POSITION_EMBEDDING_TYPE": "rope", + "PP": "1", + "PRECISION": "bf16", + "QK_LAYERNORM": "false", + "RECIPE_OVERRIDES": "{}", + "RECIPE_REQUESTED_BATCH_TOKENS": "262144", + "RECOMPUTE_GRANULARITY": "full", + "RECOMPUTE_METHOD": "uniform", + "RECOMPUTE_NUM_LAYERS": "1", + "REQUIRE_CHECKPOINT": "true", + "RESUME_POLICY": "auto", + "ROTARY_BASE": "10000", + "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847", + "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "SAVE_INTERVAL": "4000", + "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints", + "SAVE_RETAIN_INTERVAL": "4000", + "SEED": "43", + "SEQ_LENGTH": "2048", + "SWIGLU": "false", + "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard", + "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "TOKENIZER_TYPE": "HuggingFaceTokenizer", + "TP": "1", + "TRAINING_RECIPE": "paper2604_21106", + "TRAIN_ARCHITECTURE": "lookahead", + "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document", + "TRAIN_ITERS": "26703", + "UNTIE_EMBEDDINGS": "true", + "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "VOCAB_SIZE": "32001", + "WANDB_ENTITY": "LoopTF-4-CSPs", + "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "WANDB_MODE": "online", + "WANDB_PROJECT": "loopTF-workspace", + "WANDB_RESUME": "must", + "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918", + "WANDB_RUN_ID": "bapl3lfs", + "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb", + "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200", + "WARMUP_ITERS": "0", + "WEIGHT_DECAY": "0.0", + "WINDOW_ATTN_SKIP_FREQ": "1", + "WINDOW_SIZE": "none", + "WSD_DECAY_ITERS": "0" + }, + "iteration": 24000, + "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.", + "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.", + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "non_tensor_state": {}, + "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 661315072, + "trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "tensor_parallel_size": 1, + "tokenizer": { + "active_vocab_size": 32001, + "added_tokens": [ + "<|assistant_end|>" + ], + "padded_vocab_size": 32064, + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training": { + "completed_run": true, + "consumed_input_tokens": 6291456000, + "consumed_train_samples": 3072000, + "final_checkpoint": false, + "global_batch_size": 128, + "micro_batch_size": 4, + "planned_input_tokens": 7000031232, + "resumed_from_step": 12000, + "seed": 43, + "sequence_length": 2048, + "train_iters": 26703 + }, + "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace", + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..4b7323d93105beeafde639fdd5209f2d2c1d0674 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/launch_config.env @@ -0,0 +1,123 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=true +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 + +# Pinned R3 single-node objective-strength experiment. No architecture changes. +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0 +SEED=43 +NNODES=1 +GPUS_PER_NODE=8 +TP=1 +PP=1 +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +TRAIN_ITERS=26703 +EXIT_INTERVAL=16000 +SAVE_INTERVAL=4000 +SAVE_RETAIN_INTERVAL=4000 +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +EVAL_AT_START=true +FINAL_EVAL_ITERS=270 +LOG_INTERVAL=1 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false +WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300 +WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..833b4119bf804b8698b8b4c0e43bea511e64d5ed --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/manifest.json @@ -0,0 +1,33 @@ +{ + "files": [ + { + "path": "README.md", + "sha256": "4f3f67f7cd0fbcff1a600f749e19594a449c14f16d26a275d95d0792579627b7", + "size": 2352 + }, + { + "path": "config.json", + "sha256": "40aa54a4e499638a7a319a26923e2d2d1ffb881ad97ef32417ff450d96682693", + "size": 10068 + }, + { + "path": "launch_config.env", + "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26", + "size": 3907 + }, + { + "path": "model.safetensors", + "sha256": "d66f32b66871e65a15bee0dade243f1c77c5924f2732e0afe75a11b25a15fb3b", + "size": 1322796152 + }, + { + "path": "training_recipe.json", + "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6", + "size": 4352 + } + ], + "format": "safetensors_native_megatron_state_dict", + "iteration": 24000, + "note": "This manifest does not hash itself.", + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1cffda047df41f75594c9764311a5e8d7a8fa0da --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d66f32b66871e65a15bee0dade243f1c77c5924f2732e0afe75a11b25a15fb3b +size 1322796152 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..4452defdeefcb469f5d6ed08baab96f45374163b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_24000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/README.md b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bcf1b01c5941a7b852dfd78202f168306eb86462 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/README.md @@ -0,0 +1,40 @@ +# r3_b300_alpha0p0-seed43_product_20260918-170109 — step 26703 + +Native model weights from the completed Lookahead K3/LA2 alpha=0, seed=43 experiment. +Training finished at update 26703 (7000031232 input tokens); this checkpoint is the final checkpoint. +This checkpoint consumed 7000031232 input tokens. TP=1 / PP=1, DP=8. Parameter count: 661315072 total and trainable. +The H200 continuation restored the original B300 checkpoint at step 12000. The existing step_12000 folder retains its full native resume bundle. + +Training source: https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769 +Megatron revision: d046b0de593b1802c78a7664765174c031eb452c. Python 3.13.11 / PyTorch 2.14.0+cu132. +W&B: https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/bapl3lfs +Access to the code repository is required. Paper backbone source revision is separately recorded in config.json. + +## Loading + +Use the pinned repository and its dependencies, with its src directory on PYTHONPATH. +These safetensors preserve the native Megatron state-dict names and mixed dtypes; they are not an AutoModel.from_pretrained conversion. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig + +cfg = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**cfg["model_config"]), + LookaheadConfig(**cfg["lookahead_config"])) +state = load_file("model.safetensors") +state.update(cfg["non_tensor_state"]) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k[len("model."):]: v for k, v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +K3 means three complete passes including the seed. LA2 labels the existing real/pseudo design per pass. +The saved base launch_config.env differs from effective resume settings; use config.json's effective_launch_settings and constructor arguments. +Weights, configuration and integrity manifest only; optimizer, scheduler, RNG, dataloader state and training data are excluded from this export. +No final-checkpoint evaluation score is claimed here; the last periodic evaluation occurred at step26700. diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/config.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4f563fa88ba563f74933633241ae63ce99ae02d0 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/config.json @@ -0,0 +1,237 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "data_parallel_size": 8, + "effective_launch_settings": { + "ADAM_BETA1": "0.8", + "ADAM_BETA2": "0.95", + "ADAM_EPS": "1e-10", + "ATTENTION_DROPOUT": "0", + "ATTENTION_OUTPUT_GATE": "false", + "AUX_MTP_HORIZON": "1", + "AUX_MTP_LOSS_WEIGHT": "0.3", + "AUX_MTP_RANK": "128", + "BACKBONE_PROFILE": "paper2604_vanilla", + "CKPT_FORMAT": "torch", + "CLIP_GRAD": "0.0", + "DATALOADER_SAVE": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/dataloader", + "DATASET_NAME": "paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_DIR": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049", + "DATA_PREFLIGHT": "false", + "DATA_ROOT": "/root/dl/runtime/script/loopTF-workspace/data", + "DISABLE_BIAS_LINEAR": "true", + "DISABLE_WANDB": "false", + "ENABLE_RECOMPUTE": "false", + "EVAL_AT_START": "true", + "EVAL_INTERVAL": "100", + "EVAL_ITERS": "2", + "EVAL_ONLY": "false", + "EXIT_INTERVAL": "26703", + "EXIT_SIGNAL_HANDLER": "false", + "EXPECTED_INPUT_LENGTH": "2048", + "EXPECTED_RECORD_LENGTH": "2049", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "FFN_HIDDEN_SIZE": "5632", + "FINAL_EVAL_ITERS": "270", + "GLOBAL_BATCH_SIZE": "128", + "GPUS_PER_NODE": "8", + "GROUP_QUERY_ATTENTION": "false", + "HIDDEN_DROPOUT": "0", + "HIDDEN_SIZE": "1408", + "LAYERNORM_EPSILON": "0.000001", + "LOAD_PATH": "/root/dl/runtime/script/loopTF-workspace/output/checkpoint-downloads/r3_b300_alpha0p0-seed43_product_20260918-170109/step_12000/checkpoints", + "LOG_INTERVAL": "1", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.0", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LR": "0.014", + "LR_DECAY_STYLE": "linear", + "LR_WSD_DECAY_STYLE": "linear", + "MAKE_VOCAB_SIZE_DIVISIBLE_BY": "64", + "MAX_POSITION_EMBEDDINGS": "2048", + "MEGATRON_EXTRA_ARGS": "", + "MICRO_BATCH_SIZE": "4", + "MIN_LR": "0.0007470638533717787", + "MIN_MODEL_INPUT_TOKENS": "7000000000", + "MUON_MOMENTUM": "0.95", + "MUON_NESTEROV": "true", + "NORMALIZATION": "RMSNorm", + "NUM_ATTENTION_HEADS": "11", + "NUM_LAYERS": "20", + "NUM_QUERY_GROUPS": "11", + "NUM_WORKERS": "0", + "ONLINE_DATA": "true", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_DATA_CACHE_DIR": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "ONLINE_LATENCY_LOG_DIR": "", + "ONLINE_PACK_SEQUENCES": "false", + "ONLINE_PREFETCH_BATCHES": "16", + "ONLINE_RAW_MANIFEST": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", + "ONLINE_SHUFFLE_BUFFER_RECORDS": "128", + "ONLINE_SOURCE_MODE": "local", + "ONLINE_SOURCE_RETRY_LIMIT": "8", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TEXT_COLUMN": "text", + "ONLINE_TOKENIZER_BATCH_SIZE": "256", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "ONLINE_VALIDATION_MANIFEST": "", + "OPTIMIZER": "muonh", + "OUTPUT_ROOT": "/root/dl/runtime/script/loopTF-workspace/output", + "PAPER_EMBEDDING_LR": "0.3", + "PAPER_NORM_LR": "0.005", + "PAPER_OTHER_LR": "0.005", + "PAPER_REFERENCE_BATCH_TOKENS": "262144", + "PAPER_REFERENCE_TOKENS": "983315200", + "PAPER_REFERENCE_WIDTH": "640", + "PAPER_UNEMBEDDING_LR": "0.004", + "POSITION_EMBEDDING_TYPE": "rope", + "PP": "1", + "PRECISION": "bf16", + "QK_LAYERNORM": "false", + "RECIPE_OVERRIDES": "{}", + "RECIPE_REQUESTED_BATCH_TOKENS": "262144", + "RECOMPUTE_GRANULARITY": "full", + "RECOMPUTE_METHOD": "uniform", + "RECOMPUTE_NUM_LAYERS": "1", + "REQUIRE_CHECKPOINT": "true", + "RESUME_POLICY": "auto", + "ROTARY_BASE": "10000", + "RUN_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847", + "RUN_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "SAVE_INTERVAL": "4000", + "SAVE_PATH": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/checkpoints", + "SAVE_RETAIN_INTERVAL": "4000", + "SEED": "43", + "SEQ_LENGTH": "2048", + "SWIGLU": "false", + "TENSORBOARD_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/tensorboard", + "TEST_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "TOKENIZER_MODEL": "/taijifs_wzbd31/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "TOKENIZER_TYPE": "HuggingFaceTokenizer", + "TP": "1", + "TRAINING_RECIPE": "paper2604_21106", + "TRAIN_ARCHITECTURE": "lookahead", + "TRAIN_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/train_text_document", + "TRAIN_ITERS": "26703", + "UNTIE_EMBEDDINGS": "true", + "VALID_DATA_PATH": "/root/dl/runtime/script/loopTF-workspace/data/paper2604_fineweb_edu_online_llama2_32001_t2049/validation_text_document", + "VOCAB_SIZE": "32001", + "WANDB_ENTITY": "LoopTF-4-CSPs", + "WANDB_EXP_NAME": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "WANDB_MODE": "online", + "WANDB_PROJECT": "loopTF-workspace", + "WANDB_RESUME": "must", + "WANDB_RUN_GROUP": "r3_lookahead_future_grounding_20260918", + "WANDB_RUN_ID": "bapl3lfs", + "WANDB_SAVE_DIR": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_b300_alpha0p0-seed43_product_20260918-170109_h200_resume_20260919-152847/wandb", + "WANDB_TAGS": "product,resume,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300,h200", + "WARMUP_ITERS": "0", + "WEIGHT_DECAY": "0.0", + "WINDOW_ATTN_SKIP_FREQ": "1", + "WINDOW_SIZE": "none", + "WSD_DECAY_ITERS": "0" + }, + "iteration": 26703, + "launch_snapshot_note": "launch_config.env is the original base snapshot; effective_launch_settings and constructor configs capture resume overrides.", + "loading": "Use the pinned source LookaheadTransformer; remove exactly the model. key prefix; strict=True, assign=True on a meta-constructed model preserves checkpoint dtypes. Not Transformers AutoModel format.", + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "megatron_revision": "d046b0de593b1802c78a7664765174c031eb452c", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "non_tensor_state": {}, + "paper_backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 661315072, + "trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109", + "tensor_parallel_size": 1, + "tokenizer": { + "active_vocab_size": 32001, + "added_tokens": [ + "<|assistant_end|>" + ], + "padded_vocab_size": 32064, + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training": { + "completed_run": true, + "consumed_input_tokens": 7000031232, + "consumed_train_samples": 3417984, + "final_checkpoint": true, + "global_batch_size": 128, + "micro_batch_size": 4, + "planned_input_tokens": 7000031232, + "resumed_from_step": 12000, + "seed": 43, + "sequence_length": 2048, + "train_iters": 26703 + }, + "training_code_repository": "https://github.com/FFishy-git/loopTF-workspace", + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..4b7323d93105beeafde639fdd5209f2d2c1d0674 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/launch_config.env @@ -0,0 +1,123 @@ +# Look-ahead K3/LA2 matching vanilla/476M_7b_FBW0_K3.env's feedback depth and training recipe. +# LA2 names the existing two-stage design: one real plus one pseudo backbone forward per pass. +# See configs/lookahead/README.md for the additional pseudo/refiner computation. +TRAIN_ARCHITECTURE=lookahead +BACKBONE_PROFILE=paper2604_vanilla +TRAINING_RECIPE=paper2604_21106 +RECIPE_OVERRIDES='{}' +# Use the installed Transformers release; the cluster's dev checkout is incompatible. +TRANSFORMERS_PATH='' +RUN_NAME=paper2604_lookahead_476m_7b_online_K3_LA2 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false + +# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. +# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. +DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 +ONLINE_DATA=true +# Match FBW0_K3: use the complete local raw cache when available. +ONLINE_SOURCE_MODE=auto +ONLINE_DATASET=HuggingFaceFW/fineweb-edu +ONLINE_DATASET_SUBSET=sample-100BT +ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 +ONLINE_SOURCE_SPLIT=train +ONLINE_TEXT_COLUMN=text +ONLINE_VALIDATION_FRACTION=0.001 +ONLINE_SPLIT_SEED=260421106 +TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf +ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d +ONLINE_TOKENIZER_BATCH_SIZE=256 +ONLINE_SHUFFLE_BUFFER_RECORDS=128 +NUM_WORKERS=0 +DATA_PREFLIGHT=false +EXPECTED_RECORD_LENGTH=2049 +EXPECTED_INPUT_LENGTH=2048 +MIN_MODEL_INPUT_TOKENS=7000000000 + +# Match FBW0_K3's data stream: four nodes, TP=PP=1, DP=32. +# The multi-node launcher supplies the node rank; preflight can override NNODES. +NNODES=4 +GPUS_PER_NODE=8 +TP=1 +PP=1 +# Four accumulation rounds on 8 GPUs, one on 32 GPUs (TP=PP=1). +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +SEQ_LENGTH=2048 +MAX_POSITION_EMBEDDINGS=2048 +# ceil(7_000_000_000 / 262_144) = 26_703 => 7_000_031_232 physical input tokens. +TRAIN_ITERS=26703 + +NUM_LAYERS=20 +HIDDEN_SIZE=1408 +FFN_HIDDEN_SIZE=5632 +NUM_ATTENTION_HEADS=11 +NUM_QUERY_GROUPS=11 +GROUP_QUERY_ATTENTION=false +ATTENTION_OUTPUT_GATE=false +QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. +SWIGLU=false # Profile supplies squared-ReLU. +DISABLE_BIAS_LINEAR=true +NORMALIZATION=RMSNorm +LAYERNORM_EPSILON=0.000001 +HIDDEN_DROPOUT=0 +ATTENTION_DROPOUT=0 +WINDOW_SIZE=none +WINDOW_ATTN_SKIP_FREQ=1 +POSITION_EMBEDDING_TYPE=rope +ROTARY_BASE=10000 +UNTIE_EMBEDDINGS=true +TOKENIZER_TYPE=HuggingFaceTokenizer +VOCAB_SIZE=32001 +MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 +PRECISION=bf16 +ENABLE_RECOMPUTE=true # Recompute backbone layers; preserves the model and data budget. +MEGATRON_EXTRA_ARGS='' + +# Three complete real/pseudo/refiner passes on every update and during evaluation. +# Includes one seed and two feedback passes; loss = L1 + (L2 + L3) / 2. +LOOKAHEAD_MODE=parallel +LOOKAHEAD_NUM_PASSES=3 +LOOKAHEAD_FEEDBACK_LOSS_WEIGHT=1 +LOOKAHEAD_ATTENTION_BACKEND=sdpa_auto +LOOKAHEAD_CHECKPOINT_REFINER=true +LOOKAHEAD_REFINER_FFN_SIZE=5632 +LOOKAHEAD_REFINER_GATE_BIAS=-2 +WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2 +WANDB_RUN_GROUP=lookahead_476m_7b_K3_LA2 + +# Frequent logs and a short sampled evaluation for the initial training check. +WANDB_ENTITY=LoopTF-4-CSPs +WANDB_PROJECT=loopTF-workspace +WANDB_MODE=online +WANDB_RESUME=never +SAVE_INTERVAL=4000 +EVAL_AT_START=true +EVAL_INTERVAL=100 +EVAL_ITERS=2 +LOG_INTERVAL=1 +CKPT_FORMAT=torch +SEED=42 + +# Pinned R3 single-node objective-strength experiment. No architecture changes. +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0.0 +SEED=43 +NNODES=1 +GPUS_PER_NODE=8 +TP=1 +PP=1 +MICRO_BATCH_SIZE=4 +GLOBAL_BATCH_SIZE=128 +TRAIN_ITERS=26703 +EXIT_INTERVAL=16000 +SAVE_INTERVAL=4000 +SAVE_RETAIN_INTERVAL=4000 +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +EVAL_AT_START=true +FINAL_EVAL_ITERS=270 +LOG_INTERVAL=1 +RESUME_POLICY=fresh +REQUIRE_CHECKPOINT=false +WANDB_TAGS=product,r3,lookahead,k3,la2,alpha0p0-seed43,pod32xb300 +WANDB_RUN_GROUP=r3_lookahead_future_grounding_20260918 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/manifest.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..97b165affaf46399afa7d5ce7578088273b0b178 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/manifest.json @@ -0,0 +1,33 @@ +{ + "files": [ + { + "path": "README.md", + "sha256": "3ed67538dda6cad8421469dc3b1c69a82e79677c02a41b07ae48b28d04ecf876", + "size": 2337 + }, + { + "path": "config.json", + "sha256": "73e86c7512e69d623c01aa788317b26a0c8da4c3a40cb6b6e62dd580e18238fe", + "size": 10067 + }, + { + "path": "launch_config.env", + "sha256": "80859d2281685349d081b93773a1b92627e58c6b170b77d51090d928841bed26", + "size": 3907 + }, + { + "path": "model.safetensors", + "sha256": "db283fc16f1512d9fa2fe9037dc2fc1e2ac821580d426fe452e04c3f0b3eea65", + "size": 1322796152 + }, + { + "path": "training_recipe.json", + "sha256": "d3c0c45e3bd3d5c1b8fa1c7c242ded1251d6459482b1fea675e3ae19ecfccca6", + "size": 4352 + } + ], + "format": "safetensors_native_megatron_state_dict", + "iteration": 26703, + "note": "This manifest does not hash itself.", + "run_name": "r3_b300_alpha0p0-seed43_product_20260918-170109" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd9c110dc5b66a5a4a98e3da9c7c5058ed9fc91c --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db283fc16f1512d9fa2fe9037dc2fc1e2ac821580d426fe452e04c3f0b3eea65 +size 1322796152 diff --git a/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..4452defdeefcb469f5d6ed08baab96f45374163b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0-P0-F0-seed43-7b--bapl3lfs/step_26703/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/NAMING.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..bcba114566ab33cc6836e93929611320ec9bba37 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "3pjop6r1", + "previous_wandb_name": "r3-a0-r0-d0p2-p0p2-f0-s42-7b--3pjop6r1", + "training_run_name": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "canonical_name": "Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R0", + "D": 0.2, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D1-A0-R0_7b_seed42_20260919-040211" + ] +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bed21094de84f796cc7a1c288f577e45ea8b6a33 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 12000 + +Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_12000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f94ed36c5804d4d5888e4caf0f01d6501be43012 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 12000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 1536000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D1-A0-R0_7b_seed42_20260919-040211/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 3145728000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d5984b5d6d8a2b3af0eda59cd30289184710dcd5 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 12000, + "files": { + "README.md": { + "sha256": "89dad2f739311f317e0e6741ef1289c164f414538c66eb2cdac4076ed1bff026", + "size": 2770 + }, + "config.json": { + "sha256": "38f8db4ec5eb8adf40f48a4d67445a301104b44c1b8f72c3bc987111e1249b46", + "size": 7254 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "f458d4f65b83388bdb27f0ad77b69c5504ad2f46e7dfe7539071d77cc629baf7", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "c7af2fa97e82f6a5031f21bcc5ade89c8ad696ea29eca3d9e638f72cdefd010a", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_12000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9f32cb33d74b9c693cc37fd84538702ad90b8612 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f458d4f65b83388bdb27f0ad77b69c5504ad2f46e7dfe7539071d77cc629baf7 +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5ab778132c825b88f7f515a3915d732493ebf0 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_12000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f9ae9ad7df8668c8055754c5e0e9a7f63f79ef38 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 16000 + +Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_16000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..618168cfff4909527b0bb2cc063b3e5508f1acb9 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 16000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 2048000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D1-A0-R0_7b_seed42_20260919-040211/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 4194304000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6ac29056f0e71e510ef7a2437aba5a4bacdcd3c3 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 16000, + "files": { + "README.md": { + "sha256": "18ac8e398af01b6a79ebc1adaf2ac1ac18d0c0aa228dd4dc2a7a4d1dde129815", + "size": 2770 + }, + "config.json": { + "sha256": "8d0e0e1316f9d2d4b1219944a35634aa17aab12bf1fdd52c4eb07a44df203bfa", + "size": 7254 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "3fb29c5a5bb9969bd384567cf041a844e54c06cdf60c2d44f568f54991c1524c", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "c7af2fa97e82f6a5031f21bcc5ade89c8ad696ea29eca3d9e638f72cdefd010a", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_16000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e1c55664eff70bfe859bc378e5c5d276cfca378 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fb29c5a5bb9969bd384567cf041a844e54c06cdf60c2d44f568f54991c1524c +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5ab778132c825b88f7f515a3915d732493ebf0 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_16000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ad24d01dfc8bf9294affd73ca017520da4b1d0e7 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 20000 + +Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_20000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..308048b76a1ab633cacc5c21f1635d9834831a71 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 20000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2560000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,resume,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D1-A0-R0_7b_seed42_20260919-040211", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 5242880000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..cf9ed11707acfe5a53a111a594e3b5dd5fe63781 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 20000, + "files": { + "README.md": { + "sha256": "917faf5ff8bd21a87667e7b4728cfa88c669eda801ef6cae62ba6920b8e773a7", + "size": 2770 + }, + "config.json": { + "sha256": "928deb24a5613eac17b460080ed8ddb47debff8ea83bd3bec6efe6bbeca29e2e", + "size": 7200 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "6a9a4ae0a267faa5c9dad8bccbc260c4dd0877249e3de761d9cfda44dac845e9", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "dd26a09a867baf72100d3841c4e8b10b4ddf2a6ead1258f3f3dbf8ba2168911a", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_20000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8522330b63e230316c94ff64f1f584752f0fc10f --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a9a4ae0a267faa5c9dad8bccbc260c4dd0877249e3de761d9cfda44dac845e9 +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..41255b104735bb336b54284d33420948194d5a71 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_20000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aa479d1ccf81e17b5f0d214d5018f2cc4bfc725c --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 24000 + +Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_24000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b1793731cb16efa095618e32e69c58f350e8f036 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 24000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3072000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,resume,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D1-A0-R0_7b_seed42_20260919-040211", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 6291456000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b3df3491c9345c72455b51289a76e5ab074bbe5a --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 24000, + "files": { + "README.md": { + "sha256": "0c01780de77fd3b51df6551d2d77ec9ad09520fa8c6ed416a7e79112c17088c4", + "size": 2770 + }, + "config.json": { + "sha256": "99ccef92d6406361952b4d837bf689de4b5cd621e1fe1297ad6191b5c3f7e14e", + "size": 7200 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "ba13ed011d8a7ff8748b45c1f233d66605eda6872d8b00a0e72789e0530f0ad8", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "dd26a09a867baf72100d3841c4e8b10b4ddf2a6ead1258f3f3dbf8ba2168911a", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_24000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..726144a23ce190373387b5957ac96585d64a78b7 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba13ed011d8a7ff8748b45c1f233d66605eda6872d8b00a0e72789e0530f0ad8 +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..41255b104735bb336b54284d33420948194d5a71 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_24000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ebdc8bf6ba2b1ceeb679e2cb5a050e2e2e628e09 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 26703 + +Final saved checkpoint after **7,000,031,232 training input tokens**. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_26703") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available. Training and final checkpoint save completed; no final evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b84287286f6689f2baa70ea91f87dbede473d619 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/config.json @@ -0,0 +1,193 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 26703, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3417984, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": "26703", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "GLOBAL_BATCH_SIZE": "128", + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": "false", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": ".2", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.2", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": "1", + "LOOKAHEAD_PSEUDO_TOP_K": "128", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LOOKAHEAD_REFINER_VARIANT": "R0", + "MICRO_BATCH_SIZE": "8", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SEED": "42", + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": "26703", + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,resume,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 7000031232, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..04a92b69a01802caf7b5b291a781145398d806b2 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 26703, + "files": { + "README.md": { + "sha256": "546e2eb0978739565ac64c63d4347db4ee416bc4c40afeeb4d8a08ba9700e3ca", + "size": 2729 + }, + "config.json": { + "sha256": "f56c87e825b857c54ce7abc8f4f6c267fb2acd2041762980627a999aed0e2c9b", + "size": 6787 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "2e1444ff8330dfda7f1e845160e917a85b4abf32d0a60a008e33fdf1c341a371", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "dd26a09a867baf72100d3841c4e8b10b4ddf2a6ead1258f3f3dbf8ba2168911a", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_26703", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1a9dafbf239b50b2921d848516c8e6a853e20646 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e1444ff8330dfda7f1e845160e917a85b4abf32d0a60a008e33fdf1c341a371 +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..41255b104735bb336b54284d33420948194d5a71 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_26703/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..09b6444bb8903d71d8e660ef25fa988f7a0f6eb5 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 4000 + +Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_4000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2a2858e1e9c9fb843da6fcef2a35659464a0233d --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 4000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 512000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004158_previous-checkpoints-hf/metadata/r3_D1-A0-R0_7b_seed42_20260919-040211/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 1048576000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..86b434c4fba8f2f401ee7c2b30b9fad28a58b588 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 4000, + "files": { + "README.md": { + "sha256": "c1baa5f40595e65e9d47d2a7b3ee5d4c267b1321e60c8cde23d8239a8fbef574", + "size": 2767 + }, + "config.json": { + "sha256": "c1b0e3523b878984693f79b95fcd3a7b1f1fb9ee15bedf18ed587eb785c3da97", + "size": 7246 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "9a7816a26d104e7b0a2d5d9170217ffe89e554ffd4ae55556163df48c4a2b3d6", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "c7af2fa97e82f6a5031f21bcc5ade89c8ad696ea29eca3d9e638f72cdefd010a", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_4000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c6c4e58c7ffc4bda4b05d2d5ca51406959284b91 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a7816a26d104e7b0a2d5d9170217ffe89e554ffd4ae55556163df48c4a2b3d6 +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5ab778132c825b88f7f515a3915d732493ebf0 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_4000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/README.md b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b49de1cb05786784f1006b0c4faec1bd723f81cf --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/README.md @@ -0,0 +1,37 @@ +# r3_D1-A0-R0_7b_seed42_20260919-040211 — step 8000 + +Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **661,315,072 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_8000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/3pjop6r1). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/config.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..084167dd7c1f664023d00ad93a0740a1219b5aa6 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 8000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 1024000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.2, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "3pjop6r1", + "WANDB_TAGS": "product,lookahead,ablation,d1-a0-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.2, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D1-A0-R0_7b_seed42_20260919-040211/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 95170944, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 661315072, + "total_trainable": 661315072 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 143, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 661315072, + "total_trainable_parameters": 661315072, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211", + "training_tokens": 2097152000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/launch_config.env b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..ebb9370674621441e01ec5a6f3e341138db19511 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a0-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a0-r0 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/manifest.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..5b932a39039b4d680ef6c87e01ffa82d5bd80c1e --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 8000, + "files": { + "README.md": { + "sha256": "99bc921fc17eedd281909baee60b73db0cd3e70861599278ea2274326a7ceca4", + "size": 2767 + }, + "config.json": { + "sha256": "da0cfcf8faefd3225c223f4fdf3d5b881a3198164cfa314f946adbd11baf4d9f", + "size": 7253 + }, + "launch_config.env": { + "sha256": "073c2dd201ee938cac8a2d292f7fd21a11101c25259a5b02039531058d60dd2f", + "size": 387 + }, + "model.safetensors": { + "sha256": "9d6f005de6a4c7672ec47037c240923acb70a8bbacb36ce9b3443ba32f60edce", + "size": 1322796184 + }, + "training_recipe.json": { + "sha256": "c7af2fa97e82f6a5031f21bcc5ade89c8ad696ea29eca3d9e638f72cdefd010a", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D1-A0-R0_7b_seed42_20260919-040211/step_8000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D1-A0-R0_7b_seed42_20260919-040211" +} diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/model.safetensors b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..38be5b5bb2fc569ed7cc840261215ee2e3ba5d44 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6f005de6a4c7672ec47037c240923acb70a8bbacb36ce9b3443ba32f60edce +size 1322796184 diff --git a/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/training_recipe.json b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5ab778132c825b88f7f515a3915d732493ebf0 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R0-D0p2-P0p2-F0-seed42-7b--3pjop6r1/step_8000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 661315072, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 95170944, + "total_parameters": 661315072 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/NAMING.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..8ef2adf425fac6a064310d5612e64812e65728cd --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "rsk51lmo", + "previous_wandb_name": "r3-a0-r1-d0-p0-f1-s42-7b--rsk51lmo", + "training_run_name": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "canonical_name": "Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A0", + "R": "R1", + "D": 0, + "P": 0, + "F": 1, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" + ] +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b3e56da4c07c8047a854403e861e6a39e9ef4e92 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 12000 + +Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_12000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..57ecabab3a6fddbbf599ed9c1fc8156901afd5bd --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 12000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 1536000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 3145728000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..720949434d9780c8106e70db0eecc588a7854ba5 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 12000, + "files": { + "README.md": { + "sha256": "288c505d627776788e3bb01589c9c0736b8e6e56365af3d40b31f0f75a3e93e6", + "size": 2779 + }, + "config.json": { + "sha256": "2fb5cfd1d138c8f68feef4cc242b86a61d5f273cd218b891279209cb0a465a87", + "size": 7332 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "2e0ecf712296d92827e8e30f46a8ea3af16fec22215d5f364e9b09131c09442a", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_12000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f25f65b93c0e462fede92c046c9efd44c0431c34 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e0ecf712296d92827e8e30f46a8ea3af16fec22215d5f364e9b09131c09442a +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_12000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1d9eaf3ce31861bff6daf8a23c8db8c170d9128a --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 16000 + +Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_16000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1704b656d8a04545c16d9de5f5224f9f33d8f92b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 16000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2048000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 4194304000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..fbe1ce3de21fae1dd3593ac3404553bc6b958b15 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 16000, + "files": { + "README.md": { + "sha256": "17d15da629ff9b5f64ed201df3b041c73d6b78d7039658a523af4c48d829c07a", + "size": 2779 + }, + "config.json": { + "sha256": "f73e6408e940b13e8e5a05ed35b8b3a62035688e368a8c9e2efbc5c7e3c8034c", + "size": 7332 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "4b4ffb19ccee19481df96b2fd6967a2f0be723e786b086a8b866b0f7d0c04a96", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_16000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3fd7089c6037b109d7999a3bb3e1aa8562833198 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4ffb19ccee19481df96b2fd6967a2f0be723e786b086a8b866b0f7d0c04a96 +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_16000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1187b87dbe8df9fd649a3547e5ef0595bc01900f --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 20000 + +Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_20000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f53a45a57a9a3918f555f201c9f64677c87d7d42 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 20000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2560000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 5242880000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..6539655e24d8f6807d9d2854b3ea49e85901233b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 20000, + "files": { + "README.md": { + "sha256": "0a820dbabc91193c1f40c79dd98953c58e91f256ce3621b43e41280f902381be", + "size": 2779 + }, + "config.json": { + "sha256": "f735f4da06bb4669907fe727663f1f2c90b21ca138c90b2b2a00d6a99307d9f4", + "size": 7332 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "66c4951ff29d45a4b3ad3b50a11403a6d2f554619a9cc70fb7da8e030c6017d4", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_20000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9b62104d4b26cf9407e5ff3d3e5f69dfd32ac8c7 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66c4951ff29d45a4b3ad3b50a11403a6d2f554619a9cc70fb7da8e030c6017d4 +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_20000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c9ae5a18961d7a082e0d63d8ae08921c288853cc --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 24000 + +Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_24000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a29312dff0c3623e602f6d9cbf7a5229fc3f11d --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 24000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3072000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 6291456000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..18247e20cd868562c46765b9c5337c56cffb9313 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 24000, + "files": { + "README.md": { + "sha256": "0e157b498d488b3ce5ab27206ecc4be08cf079d266b7c752727ad4c2fe328735", + "size": 2779 + }, + "config.json": { + "sha256": "6242c30e6d19ba2cdc4915eac302e485a506e93a0dd0caf869dbf72a9ed7c54d", + "size": 7332 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "dba4cbd77f82f65e03593c2a0c4b68cf3c58b035124391af5ab8c84fe16d3fae", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_24000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0325475e5070a42f60fb03b4e6c52c2929a11171 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dba4cbd77f82f65e03593c2a0c4b68cf3c58b035124391af5ab8c84fe16d3fae +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_24000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f34c88bcffed5f4455682e73fb574f092938699a --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 26703 + +Final checkpoint at step **26,703**, after **7,000,031,232 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_26703") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d058c2f42465b4ba234dd3bb520be3f5748d6adf --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 26703, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3417984, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": true, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 7000031232, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8669d5b53fe7e0c14edf869f8a0f9d193f780680 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 26703, + "files": { + "README.md": { + "sha256": "62d4ed492d7cb93ff8ff8022d17b0e41db5abfd6a0c2896af6a01fd72a618589", + "size": 2772 + }, + "config.json": { + "sha256": "604d1cb5f6b24651f00ba4dfa5cda23dda31d46a5f32985178d5d03537bedba2", + "size": 7331 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "c82601be34e94a004fba843eed470d82819266f0bd8345180e9e4c910db15bc5", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_26703", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8f3546aef2971f66b9cc2f866a50fe8c0a10f62b --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c82601be34e94a004fba843eed470d82819266f0bd8345180e9e4c910db15bc5 +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_26703/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..57bc415b2c0c4506a2380e2fa52d2cb865551230 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 4000 + +Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_4000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..edca264b4be8108869199a85b99c4804931a0764 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 4000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 512000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 1048576000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8856c56d7165f3b491d9822a2630623570de22be --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 4000, + "files": { + "README.md": { + "sha256": "6ab458c433f1e66ea259c4e98454298beec8d17093c0daa078ca6979f125f0d7", + "size": 2776 + }, + "config.json": { + "sha256": "93a68c4f1b85395da625deeb76f14f04fcb2c73a72912a83b0f52207d037051a", + "size": 7330 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "d269db081640475474d6346fe00fb666be54a8733622adf7bad0c568627477ec", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_4000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4531f8033b4959bdf3d567f67e7c5c05a42fbb5a --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d269db081640475474d6346fe00fb666be54a8733622adf7bad0c568627477ec +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_4000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/README.md b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..53f209f62642ce96e986f406427933e14f52d8cb --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A0-R1_7b_seed42_20260919-161651 — step 8000 + +Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **667,258,240 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_8000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/rsk51lmo). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/config.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..52818a26ce5b98ce87fa269ba128a4989fe80fc3 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 8000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 1024000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A0", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "rsk51lmo", + "WANDB_TAGS": "product,lookahead,ablation,d0-a0-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 101114112, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 667258240, + "total_trainable": 667258240 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A0", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 195, + "tensor_dtypes": { + "torch.bfloat16": 146, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 667258240, + "total_trainable_parameters": 667258240, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651", + "training_tokens": 2097152000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/launch_config.env b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..084b3f5c875a2346d3411adc82722c273ffc8084 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a0-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A0 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a0-r1 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/manifest.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4fb2876fcb3845367730051c4566f69f429dc44f --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 8000, + "files": { + "README.md": { + "sha256": "831719409fe6a05d6a11354863eff2b65026ffd404b958c83903725a3800e3fa", + "size": 2776 + }, + "config.json": { + "sha256": "56bcc55075fb770bfa5d5ec9a416c5518481da3137d191e046fb16cb699015b5", + "size": 7331 + }, + "launch_config.env": { + "sha256": "67649480c5114ec93a6804ef65db81682e1e038436952b20ff24429899cbce1c", + "size": 523 + }, + "model.safetensors": { + "sha256": "cebb536ad846463b345a3a7c399b592511c8dd74833785499151273d3be06234", + "size": 1334676968 + }, + "training_recipe.json": { + "sha256": "952a223f00022b752ff0ac445ac65d872bb74844e23cc2ee4692ef5784831329", + "size": 4393 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651/step_8000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A0-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/model.safetensors b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8fdf23b4c6553094630689b6ccec123248d050e2 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cebb536ad846463b345a3a7c399b592511c8dd74833785499151273d3be06234 +size 1334676968 diff --git a/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/training_recipe.json b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..7ab20beb5a4f79e87822c408dba84c96eef80636 --- /dev/null +++ b/Lookahead-K3-LA2-A0-R1-D0-P0-F1-seed42-7b--rsk51lmo/step_8000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 667258240, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 101114112, + "total_parameters": 667258240 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A0", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..bd6dd92c014c5b42edf86f0024ab942f630c1813 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "1jtjk6d7", + "previous_wandb_name": "r3-a2-r0-d0-p0-f0-s42-7b--1jtjk6d7", + "training_run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A2-R0_7b_seed42_20260919-160042" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7a3467a3d84ade7c8acf17dfbcf3103420b36eca --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_12000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 12000; 3,145,728,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e40d01771d2c98d8a74a5ece14b687f823dbcdaf --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 12000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 3145728000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3b6458e620f80c6e379f0174287d5e19fecd1572 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1975, + "sha256": "e5a8c2eb337c73426182ddcf215c2b8f4a3675586d70100b2daa3c36d634ab74" + }, + "config.json": { + "size": 3804, + "sha256": "508c548c3311bf4e0bc44db0779337f998549d6d6fc1b080eef347b5349bc547" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "3c8c073355d92e9ba313cbed3ebfd61b1557319606688f657a5c30a5b6b608c5" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d493d29c039c2bad064e0efdfdc7c6d66c2805cb --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c8c073355d92e9ba313cbed3ebfd61b1557319606688f657a5c30a5b6b608c5 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_12000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ac4503df72e7c31c9e70e80fa96e1784f6543fb --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_16000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 16000; 4,194,304,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc635c619eba5fe000972d88298833a8711a532 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 16000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 4194304000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..1dab7fb5792a3855eb2c3519121791e4bae3d50b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1975, + "sha256": "3b4c16631219af90cbe664aecd244e7f9bb0b9623676e2934bb460c00429a6fb" + }, + "config.json": { + "size": 3804, + "sha256": "69ac97a6ba31e6639611ea6c90f85cbe9a7ddba8d13a5920bc53375e855fcf93" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "a17e361c7ef7fe163aa2f2546ba1061f5a5a25c25b366cd7c2a1c77e277179e4" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4fe67e7ad706664ab4f2c3ba3bd7354a82546a73 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a17e361c7ef7fe163aa2f2546ba1061f5a5a25c25b366cd7c2a1c77e277179e4 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_16000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..53e834a1812861bbe35191f88ea1c7d0bb1a77a0 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_20000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 20000; 5,242,880,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..954aaee0cd1b0167bb81cf691fc1006a3c3ced09 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 20000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 5242880000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a0c04a49fa800c47947ab3d12b4475e6bd30b3cb --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1975, + "sha256": "897f7deab7bb63748c31ee91e1c2e6622bd57ff9b81c6d50d4b3d1077bf09ebb" + }, + "config.json": { + "size": 3804, + "sha256": "ccaf2f49f0e1fcb11b05e1b906b61aa65a61a663f5b16df9e80e2b689ec5b85a" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "6a975d977f6be0ed9b9db49ec1184e6b6017363a8f189ffca826b9e929bda9e0" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b2931711870c65034d70383c0cfd26cf33b2e10a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a975d977f6be0ed9b9db49ec1184e6b6017363a8f189ffca826b9e929bda9e0 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_20000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8ec3554054d629d3c1e29198bf8f05ec3d6cdac3 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_24000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 24000; 6,291,456,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03ff6c53a5006506191c3702ff6693e9065d144b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 24000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 6291456000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e0ff1eb65214a06250131e3ac24807cdbcfcfde4 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1975, + "sha256": "8bd8ac1e34256f14add1e2b9eddb31665e8aff45f1dbc4c0c4b49f613cfdd42d" + }, + "config.json": { + "size": 3804, + "sha256": "a9d88fa72cb68dc9710407d83f9b6f0c6fac6b6871b97e15f7a036427cbe35ea" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "a3a3269cc203b15a30c5dd2fe6f488b7525ba0c6d496eaad0727b3a7b6d5b0a5" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8896ace57539edd0d5bc6c22452ed82a23a97cba --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3a3269cc203b15a30c5dd2fe6f488b7525ba0c6d496eaad0727b3a7b6d5b0a5 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_24000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..94f8271807846b4b50b80cd8565aae15752857db --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_26703 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 26703; 7,000,031,232 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c728b2521e2445ff7cbb3e78c072019893dd5599 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 26703, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 7000031232, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..110065d178e104c32e564533d74d2697fbac5710 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1975, + "sha256": "a0ebdb831c160ed2e01d102cc8271cd4e7251a54ab54efb1481f7366c58ac304" + }, + "config.json": { + "size": 3804, + "sha256": "65e6402c0ea98d127497f678c696d993c820cead4503ee202942d48aa6cda543" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "5ab090d6ac96f22c716cfb7110f3b208c1bb3322b05846dc0a6e96259a82f91a" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a47bd53477ffcf3ce8a8b208ca810ead71b6c646 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ab090d6ac96f22c716cfb7110f3b208c1bb3322b05846dc0a6e96259a82f91a +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_26703/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b168177d90da53ddd0dc9f9104839c9de34a9b7 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_4000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 4000; 1,048,576,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2a6794cba52bab1e16018e36b014053fecf51ed8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 4000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 1048576000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4469133ba108e7f5f97affca1f8745d7d1c7602c --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1973, + "sha256": "9ffb2ae8fdc4fc7edaa31ffc1cae0ecd229016dc1e3cbc0bc97cf583244554db" + }, + "config.json": { + "size": 3803, + "sha256": "7da456f38679f9749a26028b5e1298aba6340d9fc11e7dca8664b8bcdcb37f72" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "2bb1d38864ad268cc88993d119e2e0c5d4ebbfea999484fe3247cf1951d3033a" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d5393c3a8d1f4a5989f54f50e45bbf1cbee3dfcc --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb1d38864ad268cc88993d119e2e0c5d4ebbfea999484fe3247cf1951d3033a +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_4000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a29454d047bfc7140a3a76dff519fa3f7b3d908 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/README.md @@ -0,0 +1,26 @@ +# r3_D0-P0-A2-R0_7b_seed42_20260919-160042 / step_8000 + +Native Lookahead A2/R0, 659,332,608 total and 659,332,608 trainable parameters. Step 8000; 2,097,152,000 consumed input tokens. Full run finished at 26,703 steps / 7,000,031,232 input tokens; earlier steps are intermediate weights. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/037edaf25d7ffbc44527a0322f077a675d6ac628 (repository access may be required). Install this revision and its training dependencies. The backbone source revision is recorded separately in config.json. Source W&B identity: LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7. + +Weights/configuration only: native Megatron state-dict safetensors, not a Transformers AutoModel package or a full training-resume bundle. Optimizer/scheduler/RNG/dataloader states are excluded. K3 includes the seed pass. Saved draft loss=0.0, future loss=0.0, refined-future loss=0.0; constructor options are pinned in config.json. + +Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k,v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +The pinned Llama-2 tokenizer includes the assistant_end token. All exported tensor names/shapes/dtypes/values were compared exactly with the trusted checkpoint; weights are finite and strict native loading passed. manifest.json hashes each payload file except itself. Source checkpoints are retained. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ffabdac69147a1594b3e5e7c4d456fb0e7bb4225 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/config.json @@ -0,0 +1,105 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_D0-P0-A2-R0_7b_seed42_20260919-160042", + "iteration": 8000, + "checkpoint_version": 3.0, + "training_code_revision": "037edaf25d7ffbc44527a0322f077a675d6ac628", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "native mixed bfloat16/float32 tensor dtypes preserved", + "training_tokens": 2097152000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.0, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original reviewed base launch snapshot; effective saved overrides and constructors are recorded in config.json.", + "non_tensor_state": {}, + "provenance": { + "wandb_run": "LoopTF-4-CSPs/loopTF-workspace/1jtjk6d7", + "state": "finished", + "final_step": 26703, + "source": "local checkpoint" + }, + "training_completion": "Run finished at configured step 26703, verified by log/W&B state and final checkpoint. Earlier steps are intermediate weights; no evaluation metric asserted.", + "loading": "Use pinned LookaheadTransformer; remove exactly model. prefix; strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..a83463020155858b783e01cd4e2e1405dcf75fee --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1973, + "sha256": "924ff513ffd3db73acf5b157afca37b0ae27cc6b848a5ce64358d020a4aed53a" + }, + "config.json": { + "size": 3803, + "sha256": "380b68959033ffddd80b76c2d4fb0cfc285caaec200d3086091daf36ba02e967" + }, + "launch_config.env": { + "size": 386, + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "b93db87ff1809c98092376eaebda33b3c636a5e2ad757c6624468772315e476b" + }, + "training_recipe.json": { + "size": 4391, + "sha256": "6c5f53ab1c4d25ca121732024d2acf56301ca9ef007299893363ab360dddb3f1" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87ccdfd90d837712babad4c746d78b9373678499 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b93db87ff1809c98092376eaebda33b3c636a5e2ad757c6624468772315e476b +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..b31c8393b2774dc51a0bea4f8f7f5a766442e3ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-7b--1jtjk6d7/step_8000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..b51579c4da11331f52fb4e9b9ce15860c56abaaf --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "my34axon", + "previous_wandb_name": "r4_la_hf_warmstart_s12000_to26703", + "training_run_name": "r4_la_hf_warmstart_s12000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s12000_to26703" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d62ca6759ed4b177d0228296f9a7056dca9dc310 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/README.md @@ -0,0 +1,11 @@ +# R4 Look-ahead final weights (12000 warm start) + +This is the final step 26,703 weights-only export for `r4_weights_only_d0_p0_a2_r0_s12000_to26703`, warm-started from the FBW export at step 12000. + +## Format + +`model.safetensors` contains the native Look-ahead Megatron model state only. It excludes optimizer, scheduler, RNG, dataloader, caches, logs, and credentials. It is not a Transformers `AutoModel.from_pretrained` checkpoint. + +To load, use the code revision in `config.json`, construct `lookahead_tf.hf.model.LookaheadTransformer` with `model_config` and `lookahead_config`, remove exactly the `model.` prefix from each tensor key, and call `load_state_dict(..., strict=True, assign=True)`. + +Use `manifest.json` to verify payload hashes. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..546e904adb46f3e62243024eaf6de658cfea983b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/config.json @@ -0,0 +1,1041 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "excluded_state": [ + "optimizer", + "opt_param_scheduler", + "rng_state", + "dataloader" + ], + "iteration": 26703, + "loading": { + "format": "native PyTorch state dict saved with safetensors; not a Transformers AutoModel", + "instructions": "Construct LookaheadTransformer(model_config, lookahead_config), remove exactly the model. prefix, then load_state_dict(..., strict=True, assign=True).", + "key_prefix": "model." + }, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 0.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "model_arguments": { + "_is_global_batch_size_explicitly_specified": true, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "accumulate_allreduce_grads_in_fp32": true, + "activation_func_clamp_value": null, + "activation_offload_fraction": 1.0, + "adam_beta1": 0.8, + "adam_beta2": 0.95, + "adam_eps": 1e-10, + "adaptive_muon_beta2": 0.95, + "adaptive_muon_eps": 1e-10, + "adaptive_muon_moment2_method": "normuon", + "add_bias_linear": false, + "add_position_embedding": true, + "add_qkv_bias": false, + "adlr_autoresume": false, + "adlr_autoresume_interval": 1000, + "align_grad_reduce": true, + "align_param_gather": false, + "allow_ambiguous_pad_tokens": false, + "app_tag_run_name": null, + "app_tag_run_version": "0.0.0", + "apply_query_key_layer_scaling": false, + "apply_residual_connection_post_layernorm": false, + "apply_rope_fusion": false, + "apply_wd_to_qk_layernorm": false, + "architecture": "lookahead", + "architecture_module": "[]", + "async_ckpt_cpu_priority": 10, + "async_ckpt_io_priority": 3, + "async_ckpt_use_cpu_shm": false, + "async_save": false, + "async_strategy": "mcore", + "attention_backend": "AttnBackend.auto", + "attention_cp_layout": "zigzag", + "attention_dropout": 0.0, + "attention_output_gate": false, + "attention_softmax_in_fp32": false, + "auto_detect_ckpt_format": false, + "backbone_metadata": "{'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}", + "backbone_profile": "paper2604_vanilla", + "barrier_with_L1_time": true, + "batch_invariant_backend": "te_native", + "batch_invariant_collective": "ordered", + "batch_invariant_mode": false, + "bert_binary_head": true, + "bert_embedder_type": "megatron", + "bert_load": null, + "bf16": true, + "bias_dropout_fusion": false, + "bias_gelu_fusion": false, + "bias_swiglu_fusion": true, + "biencoder_projection_dim": 0, + "biencoder_shared_query_context_model": false, + "block_data_path": null, + "cache_mla_latents": false, + "calc_ft_timeouts": false, + "calculate_per_token_loss": false, + "chat_template": null, + "check_for_large_grads": false, + "check_for_nan_in_loss_and_grad": true, + "check_for_spiky_loss": false, + "check_weight_hash_across_dp_replicas_interval": null, + "ckpt_assume_constant_structure": false, + "ckpt_convert_format": null, + "ckpt_convert_save": null, + "ckpt_convert_update_legacy_dist_opt_format": false, + "ckpt_drop_redundant_extra_state": false, + "ckpt_format": "torch", + "ckpt_fully_parallel_load": false, + "ckpt_fully_parallel_load_exchange_algo": "broadcast", + "ckpt_fully_parallel_load_per_rank_objects": false, + "ckpt_fully_parallel_load_process_group": "dp", + "ckpt_fully_parallel_save": true, + "ckpt_fully_parallel_save_deprecated": false, + "ckpt_fully_parallel_save_process_group": "dp", + "ckpt_load_validate_sharding_integrity": true, + "ckpt_pg_tensors_cache_create": false, + "ckpt_pg_tensors_cache_path": null, + "ckpt_step": null, + "classes_fraction": 1.0, + "clip_grad": 0.0, + "clone_scatter_output_in_embedding": true, + "config_logger_dir": "", + "consumed_train_samples": 3417984, + "consumed_valid_samples": 68352, + "context_parallel_size": 1, + "cp_comm_type": "['p2p']", + "cpu_offloading_num_layers": 0, + "cpu_offloading_retain_pinned_cpu_buffers": false, + "create_all_gather_group": false, + "create_attention_mask_in_dataloader": false, + "cross_entropy_fusion_impl": "native", + "cross_entropy_loss_fusion": false, + "csa_compress_ratios": null, + "csa_compress_rotary_base": 40000.0, + "csa_dense_mode": false, + "csa_window_size": 128, + "cuda_graph_impl": "none", + "cuda_graph_modules": "[]", + "cuda_graph_warmup_steps": 3, + "curr_iteration": 26702, + "data_args_path": null, + "data_cache_path": null, + "data_parallel_random_init": false, + "data_parallel_sharding_strategy": "optim_grads_params", + "data_parallel_size": 16, + "data_path": null, + "data_per_class_fraction": 1.0, + "data_sharding": true, + "dataloader_defer_npy_index_mmap": false, + "dataloader_fast_cache_load": false, + "dataloader_inter_document_masking": false, + "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s12000_to26703/dataloader", + "dataloader_type": "external", + "ddp_average_in_collective": false, + "ddp_bucket_size": null, + "ddp_num_buckets": null, + "ddp_pad_buckets_for_high_nccl_busbw": false, + "ddp_param_name_patterns_for_fp32_local_accumulation": "[]", + "ddp_reduce_scatter_with_fp32_accumulation": false, + "decode_only_cuda_graphs": false, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "decoder_num_layers": null, + "decoder_seq_length": null, + "decoupled_lr": null, + "decoupled_min_lr": null, + "decrease_batch_size_if_needed": false, + "defer_embedding_wgrad_compute": false, + "delay_offload_until_cuda_graph": false, + "delay_wgrad_compute": false, + "delta_offload_bytes_across_pp_ranks": 0, + "deprecated_use_mcore_models": false, + "deterministic_mode": false, + "dino_bottleneck_size": 256, + "dino_freeze_last_layer": 1, + "dino_head_hidden_size": 2048, + "dino_local_crops_number": 10, + "dino_local_img_size": 96, + "dino_norm_last_layer": false, + "dino_teacher_temp": 0.07, + "dino_warmup_teacher_temp": 0.04, + "dino_warmup_teacher_temp_epochs": 30, + "disable_bf16_reduced_precision_matmul": false, + "disable_jit_fuser": false, + "disable_msc_deprecated": false, + "disable_straggler_on_startup": false, + "disable_symmetric_registration": false, + "dist_ckpt_format_deprecated": null, + "dist_ckpt_optim_fully_reshardable": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_strictness": "assume_ok_unexpected", + "dist_ckpt_workers": 1, + "distrib_optim_fully_reshardable_mem_efficient": false, + "distribute_saved_activations": false, + "distributed_backend": "nccl", + "distributed_timeout_minutes": 10, + "distributed_timeout_seconds_after_init": null, + "do_test": 0, + "do_train": 1, + "do_valid": 1, + "dsa_indexer_head_dim": null, + "dsa_indexer_k_norm_epsilon": null, + "dsa_indexer_k_norm_fp32": false, + "dsa_indexer_loss_coeff": null, + "dsa_indexer_n_heads": null, + "dsa_indexer_rope_interleaved": false, + "dsa_indexer_rotate_activation": true, + "dsa_indexer_scoring_relu": true, + "dsa_indexer_skip_topk_offset": 0, + "dsa_indexer_topk": null, + "dsa_indexer_topk_freq": 1, + "dsa_indexer_use_sparse_loss": false, + "dsa_kernel_backend": "none", + "dump_param_to_param_group_map": null, + "embedding_init_method_std": null, + "embedding_path": null, + "empty_unused_memory_level": 0, + "enable_chunked_prefill": false, + "enable_cuda_graph": false, + "enable_experimental": false, + "enable_ft_package": false, + "enable_full_sharding_in_hsdp": false, + "enable_mhc_connections": false, + "enable_msc": false, + "enable_one_logger": true, + "encoder_num_layers": 20, + "encoder_seq_length": 2048, + "end_weight_decay": 0.0, + "eod_mask_loss": false, + "ep_overlap_early_attn_memory_release": false, + "error_injection_rate": 0, + "error_injection_type": "transient_error", + "eval_at_start": true, + "eval_global_batch_size": 128, + "eval_interval": 100, + "eval_iters": 2, + "eval_micro_batch_size": 8, + "evaluation_metadata": "{'eval_at_start': True, 'initial_iters': 2, 'intermediate_iters': 2, 'final_validation_iters': 270, 'test_iters': 0, 'global_batch_size': 128, 'sequence_length': 2048, 'intermediate_input_tokens': 524288, 'final_validation_sequences': 34560, 'final_validation_input_tokens': 70778880, 'validation_iterator': 'continues_existing_held_out_iterator'}", + "evidence_data_path": null, + "exit_duration_in_mins": null, + "exit_interval": 26703, + "exit_on_missing_checkpoint": true, + "exit_signal": 15, + "exit_signal_handler": false, + "exit_signal_handler_for_dataloader": false, + "exit_signal_handler_for_training": false, + "exp_avg_dtype": "torch.float32", + "exp_avg_sq_dtype": "torch.float32", + "experimental_attention_variant": null, + "expert_data_parallel_sharding_strategy": null, + "expert_gtp_weight_remat_size": 1, + "expert_model_parallel_size": 1, + "expert_tensor_parallel_num_weight_shards": 1, + "expert_tensor_parallel_size": 1, + "external_cuda_graph": false, + "fake_process_group": false, + "fault_injector_delay_start_iteration": null, + "fault_injector_fault_delay": null, + "fault_injector_fault_probabilities": null, + "fault_injector_fault_types": null, + "fault_injector_mtti_seconds": null, + "fault_injector_num_ranks": null, + "fault_injector_offset_seconds": null, + "fault_injector_ranks": null, + "fault_injector_seed": null, + "feedback_normuon_beta2": 0.95, + "feedback_scalar_lr": 0.0005, + "feedback_scalar_weight_decay": 0.0, + "feedback_z_loss_coefficient": 0.0, + "feedback_z_loss_start_fraction": 1.0, + "ffn_hidden_size": 5632, + "fim_data": false, + "fim_eod_token": "<|endoftext|>", + "fim_fragment_rate": null, + "fim_middle_token": "", + "fim_no_prefix": null, + "fim_pad_token": "", + "fim_prefix_token": "", + "fim_rate": 0.5, + "fim_split_sample": null, + "fim_spm_rate": 0.5, + "fim_suffix_token": "", + "final_eval_iters": 270, + "fine_grained_activation_offloading": false, + "fine_grained_offloading_max_inflight_offloads": null, + "finetune": false, + "first_last_layers_bf16": false, + "flash_attention_version": null, + "flash_decode": false, + "flight_recorder_dump_on_timeout": true, + "flight_recorder_dump_path": null, + "flight_recorder_extra_dump_on_exec": true, + "flight_recorder_include_only_active": true, + "flight_recorder_include_stack_trace": false, + "flight_recorder_trace_buffer_size": 2000, + "fp16": false, + "fp16_lm_cross_entropy": false, + "fp32_residual_connection": false, + "fp4": null, + "fp4_param_gather": false, + "fp4_quantizer_factory": null, + "fp4_recipe": "nvfp4", + "fp8": null, + "fp8_amax_compute_algo": "most_recent", + "fp8_amax_history_len": 1, + "fp8_interval": 1, + "fp8_margin": 0, + "fp8_output_proj": false, + "fp8_param_gather": false, + "fp8_quantizer_factory": null, + "fp8_recipe": "delayed", + "fp8_wgrad": true, + "freeze_all_layers": false, + "fsdp_db_use_persist_buf_on_alloc_fail": false, + "fsdp_double_buffer": false, + "fsdp_manual_registration": false, + "ft_num_warmup_iters": 5, + "full_validation": false, + "fused_residual_rmsnorm": false, + "gdp_cutedsl_kernel": false, + "gdp_num_chunk_states_to_recompute": 2, + "gdp_num_householder": 3, + "global_batch_size": 128, + "glu_linear_offset": 0.0, + "gpu_sniff_test_interval": null, + "grad_reduce_in_bf16": false, + "gradient_accumulation_fusion": false, + "gradient_reduce_div_fusion": true, + "group_query_attention": false, + "grpo_clamp_eps_lower": 0.01, + "grpo_clamp_eps_upper": 0.01, + "grpo_entropy_term_weight": 0.0, + "grpo_filter_groups_with_same_reward": false, + "grpo_group_size": 2, + "grpo_iterations": 2, + "grpo_kl_beta": 0.001, + "grpo_prompts_per_step": 32, + "gtp_expert_remat_nccl_ub": false, + "gtp_remat_nccl_ub": false, + "gtp_remat_opt_in_modules": "[]", + "gtp_remat_reduce_scatter_with_fp32_accumulation": false, + "gtp_weight_remat_size": 1, + "head_lr_mult": 1.0, + "heterogeneous_layers_config_encoded_json": null, + "heterogeneous_layers_config_path": null, + "hfsdp_param_gather_overlap": false, + "hidden_dropout": 0.0, + "hidden_size": 1408, + "hierarchical_context_parallel_sizes": null, + "high_priority_a2a_comm_stream": false, + "high_priority_stream_groups": "[]", + "hybrid_context_parallel": false, + "hybrid_layer_pattern": null, + "hybrid_override_pattern": null, + "hysteresis": 2, + "ict_head_size": null, + "ict_load": null, + "img_h": 224, + "img_w": 224, + "indexer_batch_size": 128, + "indexer_log_interval": 1000, + "inference_batch_times_seqlen_threshold": -1, + "inference_coordinator_port": null, + "inference_cuda_graph_all_prefills": false, + "inference_cuda_graph_max_tokens": 512, + "inference_cuda_graph_scope": "InferenceCudaGraphScope.none", + "inference_disable_ep_consensus": false, + "inference_disable_triton_nvls_kernels": false, + "inference_dynamic_batching": false, + "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, + "inference_dynamic_batching_async_sched_mode": "async", + "inference_dynamic_batching_block_size": 256, + "inference_dynamic_batching_buffer_size_gb": 40.0, + "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, + "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", + "inference_dynamic_batching_enable_prefix_caching": false, + "inference_dynamic_batching_logprobs_mode": "raw_logprobs", + "inference_dynamic_batching_mamba_memory_ratio": null, + "inference_dynamic_batching_max_requests": null, + "inference_dynamic_batching_max_tokens": null, + "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", + "inference_dynamic_batching_media_cache_routing_weight": 1.0, + "inference_dynamic_batching_num_cuda_graphs": 16, + "inference_dynamic_batching_paused_buffer_size_gb": null, + "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, + "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", + "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", + "inference_dynamic_batching_prefix_caching_mamba_gb": null, + "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, + "inference_dynamic_batching_sampling_backend": "torch", + "inference_dynamic_batching_track_generated_token_events": false, + "inference_dynamic_batching_track_paused_request_events": false, + "inference_dynamic_batching_unified_memory_level": 0, + "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, + "inference_flashinfer_mxfp8_token_capacity": null, + "inference_fuse_tp_communication": false, + "inference_grouped_gemm_backend": "vllm", + "inference_logging_step_interval": 0, + "inference_max_requests": 8, + "inference_max_seq_length": 2560, + "inference_moe_disable_fused_quant_kernels": false, + "inference_moe_token_dispatcher_type": "nvls", + "inference_rng_tracker": false, + "inference_shards": null, + "inference_text_gen_server_logging": false, + "inference_use_synchronous_zmq_collectives": false, + "inference_wandb_logging": false, + "init_method_std": 0.02, + "init_method_xavier_uniform": false, + "init_model_with_meta_device": false, + "initial_loss_scale": 4294967296, + "inprocess_active_world_size": 16, + "inprocess_barrier_timeout": 120, + "inprocess_completion_timeout": 120, + "inprocess_empty_cuda_cache": false, + "inprocess_granularity": "node", + "inprocess_hard_timeout": 90, + "inprocess_heartbeat_interval": 30, + "inprocess_heartbeat_timeout": 60, + "inprocess_last_call_wait": 1, + "inprocess_max_iterations": null, + "inprocess_monitor_process_interval": 1.0, + "inprocess_monitor_thread_interval": 1.0, + "inprocess_progress_watchdog_interval": 1.0, + "inprocess_restart": false, + "inprocess_soft_timeout": 60, + "inprocess_termination_grace_time": 1, + "is_hybrid_model": false, + "iter_per_epoch": 1250, + "iteration": 12000, + "iterations_to_skip": "[]", + "keep_fp8_transpose_cache": false, + "kitchen_attention_backend": "sdpa", + "kv_channels": 128, + "kv_lora_rank": 32, + "langrl_env_config": null, + "layernorm_epsilon": 1e-06, + "layernorm_zero_centered_gamma": false, + "lazy_mpu_init": false, + "linear_attention_freq": null, + "linear_conv_kernel_dim": 4, + "linear_cp_layout": "zigzag", + "linear_cp_mode": "headwise", + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "lion_beta1": 0.95, + "lion_beta2": 0.98, + "load": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/transfers/hf_weights_only/s12000/lookahead/checkpoints", + "load_main_params_from_ckpt": false, + "local_rank": 0, + "log_device_memory_used": false, + "log_energy": false, + "log_interval": 1, + "log_loss_scale_to_tensorboard": true, + "log_max_attention_logit": false, + "log_memory_interval": null, + "log_memory_to_tensorboard": false, + "log_num_zeros_in_grad": false, + "log_params_norm": false, + "log_progress": false, + "log_straggler": false, + "log_throughput": false, + "log_timers_to_tensorboard": true, + "log_validation_ppl_to_tensorboard": false, + "log_world_size_to_tensorboard": false, + "logging_level": null, + "logit_dtype": null, + "logits_load_decode_threads": 4, + "logits_load_dir": null, + "logits_load_ignore_errors": false, + "logits_load_kd_loss_alpha": 1.0, + "logits_load_msc_prefetch_depth": 2, + "logits_load_prefetch_factor": 3, + "logits_save_dir": null, + "logits_save_dtype": "fp16", + "logits_save_top_k": null, + "logits_save_top_p": null, + "logits_save_top_p_min_k": 1, + "lookahead_allow_objective_change_on_resume": false, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_cache_layers": null, + "lookahead_checkpoint_refiner": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_full_attention_interval": 6, + "lookahead_future_gradient_mode": "all", + "lookahead_future_loss_weight": 0.0, + "lookahead_gradient_checkpointing": false, + "lookahead_mode": "parallel", + "lookahead_normalize_feedback_embedding": true, + "lookahead_normalize_feedback_output": true, + "lookahead_num_passes": 3, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_temperature": 1.0, + "lookahead_pseudo_top_k": 128, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_z_loss_coefficient": 0.0, + "loss_scale": null, + "loss_scale_window": 1000, + "lr": 0.007470638533717786, + "lr_decay_iters": null, + "lr_decay_samples": null, + "lr_decay_style": "linear", + "lr_warmup_fraction": null, + "lr_warmup_init": 0.0, + "lr_warmup_iters": 0, + "lr_warmup_samples": 0, + "lr_wsd_decay_iters": 0, + "lr_wsd_decay_samples": null, + "lr_wsd_decay_style": "linear", + "main_grads_dtype": "torch.float32", + "main_params_dtype": "torch.float32", + "make_vocab_size_divisible_by": 64, + "mamba_head_dim": 64, + "mamba_inference_conv_states_dtype": "torch.bfloat16", + "mamba_inference_ssm_states_dtype": "torch.bfloat16", + "mamba_num_groups": 8, + "mamba_num_heads": null, + "mamba_state_dim": 128, + "mamba_training_ssm_states_dtype": null, + "manual_gc": false, + "manual_gc_eval": true, + "manual_gc_interval": 0, + "mask_factor": 1.0, + "mask_prob": 0.15, + "mask_type": "random", + "masked_softmax_fusion": true, + "max_position_embeddings": 2048, + "max_seqlen_per_dp_cp_rank": null, + "max_tokens_to_oom": 12000, + "megatron_fsdp_enable_fine_grained_param_gather": false, + "megatron_fsdp_grad_comm_dtype": null, + "megatron_fsdp_main_grads_dtype": null, + "megatron_fsdp_main_params_dtype": "torch.float32", + "megatron_fsdp_max_pool_double_buffer": false, + "megatron_fsdp_version": 1, + "megatron_lm_path": "/root/dl/runtime/Megatron-LM", + "memory_snapshot_path": "snapshot.pickle", + "merge_file": null, + "metadata_path": null, + "mhc_fused_backend": "auto", + "mhc_init_gating_factor": 0.01, + "mhc_num_residual_streams": 4, + "mhc_recompute_layer_num": null, + "mhc_sinkhorn_iterations": 20, + "micro_batch_size": 8, + "microbatch_group_size_per_vp_stage": null, + "mid_level_dataset_surplus": 0.005, + "min_loss_scale": 1.0, + "min_lr": 0.0007470638533717787, + "min_offloaded_tensor_size": 1048576, + "mla_down_proj_fusion": false, + "mlp_chunks_for_prefill": 1, + "mlp_chunks_for_training": 1, + "mmap_bin_files": true, + "mock_data": false, + "moe_apply_probs_on_input": false, + "moe_aux_loss_coeff": 0.0, + "moe_combine_bwd_dtype": "bf16", + "moe_deepep_num_sms": null, + "moe_dispatch_fwd_dtype": "bf16", + "moe_enable_deepep": false, + "moe_enable_routing_replay": false, + "moe_expert_capacity_factor": null, + "moe_expert_rank_capacity_factor": null, + "moe_ffn_hidden_size": null, + "moe_flex_dispatcher_backend": "deepep", + "moe_flex_dispatcher_num_sms": null, + "moe_grouped_gemm": false, + "moe_hybridep_num_blocks_permute": null, + "moe_hybridep_num_blocks_unpermute": null, + "moe_hybridep_num_sms": null, + "moe_hybridep_num_sms_preprocessing": 108, + "moe_hybridep_pad_uneven_dispatch_inputs": false, + "moe_input_jitter_eps": null, + "moe_latent_size": null, + "moe_layer_freq": 1, + "moe_layer_recompute": false, + "moe_mlp_glu_interleave_size": null, + "moe_ncclep_zero_copy": false, + "moe_pad_expert_input_to_capacity": false, + "moe_pad_experts_for_cuda_graph_inference": false, + "moe_paged_stash": false, + "moe_paged_stash_buffer_size_factor_cpu": 0.0, + "moe_paged_stash_buffer_size_factor_cuda": 1.1, + "moe_paged_stash_page_size": 64, + "moe_per_layer_logging": false, + "moe_permute_fusion": false, + "moe_permute_fusion_into_hybridep": false, + "moe_router_bias_update_rate": 0.001, + "moe_router_dtype": null, + "moe_router_enable_expert_bias": false, + "moe_router_force_biased": null, + "moe_router_force_load_balancing": false, + "moe_router_fusion": false, + "moe_router_group_topk": null, + "moe_router_load_balancing_type": "aux_loss", + "moe_router_num_groups": null, + "moe_router_padding_for_fp8": false, + "moe_router_padding_for_quantization": false, + "moe_router_pre_softmax": false, + "moe_router_quantile_balancing_ema": 0.0, + "moe_router_score_function": "softmax", + "moe_router_skip_muon": false, + "moe_router_topk": 2, + "moe_router_topk_scaling_factor": null, + "moe_routing_trace_capture_hidden_states": false, + "moe_routing_trace_capture_logits": false, + "moe_routing_trace_dump_weights": false, + "moe_routing_trace_max_training_iters": null, + "moe_routing_trace_path": null, + "moe_shared_expert_gate": false, + "moe_shared_expert_glu_interleave_size": null, + "moe_shared_expert_intermediate_size": null, + "moe_shared_expert_overlap": false, + "moe_single_grouped_bias": false, + "moe_single_grouped_weight": false, + "moe_token_dispatcher_type": "allgather", + "moe_token_drop_policy": "probs", + "moe_upcycling_granularity": 1, + "moe_use_grouped_tensor": false, + "moe_use_norm_before_up_proj": false, + "moe_use_upcycling": false, + "moe_z_loss_coeff": null, + "mrope_section": null, + "mscale": 1.0, + "mscale_all_dim": 0.0, + "mtp_detach_heads": false, + "mtp_hsm": false, + "mtp_hybrid_override_pattern": null, + "mtp_loss_scaling_factor": 0.1, + "mtp_num_layers": null, + "mtp_standalone": false, + "mtp_use_repeated_layer": false, + "multi_latent_attention": false, + "multiple_validation_sets": false, + "muon_coefficient_type": "quintic", + "muon_extra_scale_factor": 1.0, + "muon_fp32_matmul_prec": "medium", + "muon_momentum": 0.95, + "muon_nesterov": true, + "muon_num_ns_steps": 5, + "muon_scalar_optimizer": "adam", + "muon_scale_mode": "spectral", + "muon_split_qkv": true, + "muon_tp_mode": "duplicated", + "muon_use_syrk": false, + "mup_attn_scale_power": 1.0, + "mup_base_head_dim": null, + "mup_base_hidden_size": null, + "mup_embedding_mult": 1.0, + "mup_output_mult": 1.0, + "mup_width_mult": 1.0, + "nccl_all_reduce_for_prefill": false, + "nccl_communicator_config_path": null, + "nccl_ub": false, + "no_load_optim": null, + "no_load_rng": null, + "no_persist_layer_norm": false, + "no_rope_freq": null, + "no_save_optim": null, + "no_save_rng": null, + "no_weight_decay_cond_type": null, + "non_persistent_ckpt_type": null, + "non_persistent_global_ckpt_dir": null, + "non_persistent_local_ckpt_algo": "fully_parallel", + "non_persistent_local_ckpt_dir": null, + "non_persistent_save_interval": null, + "normalization": "RMSNorm", + "null_tokenizer_eod_id": null, + "null_tokenizer_pad_id": -1, + "num_attention_heads": 11, + "num_channels": 3, + "num_classes": 1000, + "num_dataset_builder_threads": 1, + "num_distributed_optimizer_instances": 1, + "num_experts": null, + "num_floating_point_operations_so_far": 0, + "num_layers": 20, + "num_layers_at_end_in_bf16": 1, + "num_layers_at_start_in_bf16": 1, + "num_layers_per_virtual_pipeline_stage": null, + "num_query_groups": 11, + "num_speculative_tokens": 0, + "num_virtual_stages_per_pipeline_rank": null, + "num_workers": 0, + "nvtx_ranges": false, + "object_storage_cache_path": null, + "offload_modules": "[]", + "offset_sampling_seed_by_dp_rank": true, + "one_logger_async": false, + "one_logger_project": "megatron-lm", + "one_logger_run_name": null, + "online_data": true, + "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "online_dataset": "HuggingFaceFW/fineweb-edu", + "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "online_dataset_subset": "sample-100BT", + "online_latency_log_dir": null, + "online_pack_sequences": false, + "online_prefetch_batches": 16, + "online_raw_manifest": null, + "online_shuffle_buffer_records": 128, + "online_source_mode": "stream", + "online_source_split": "train", + "online_split_seed": 260421106, + "online_text_column": "text", + "online_tokenizer_batch_size": 256, + "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "online_validation_fraction": 0.001, + "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json", + "onnx_safe": null, + "openai_gelu": false, + "optimizer": "muonh", + "optimizer_cpu_offload": false, + "optimizer_cuda_graph": false, + "optimizer_offload_fraction": 1.0, + "otel_enabled": false, + "otel_service_name": null, + "otel_span_groups": null, + "outer_dp_sharding_strategy": "no_shard", + "output_bert_embeddings": false, + "overlap_cpu_optimizer_d2h_h2d": false, + "overlap_dispatch_backward_with_experts_wgrad": false, + "overlap_grad_reduce": false, + "overlap_moe_expert_parallel_comm": false, + "overlap_p2p_comm": false, + "overlap_p2p_comm_warmup_flush": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "override_ckpt_iteration": null, + "override_opt_param_scheduler": false, + "pad_vocab_size": true, + "padded_vocab_size": 32064, + "paper_base_matrix_lr": 0.014, + "paper_embedding_lr": 0.3, + "paper_norm_lr": 0.005, + "paper_other_lr": 0.005, + "paper_reference_batch_tokens": 262144, + "paper_reference_tokens": 983315200, + "paper_reference_width": 640, + "paper_transfer": "Paper2604Transfer(width=1408, total_tokens=7000031232, batch_tokens=262144, reference_width=640, reference_tokens=983315200, reference_batch_tokens=262144)", + "paper_unembedding_lr": 0.004, + "params_dtype": "torch.bfloat16", + "patch_dim": 16, + "per_dataset_sequences_path": null, + "per_split_data_args_path": null, + "perform_initialization": true, + "perform_rl_step": false, + "phase_transition_iterations": null, + "pin_cpu_grads": true, + "pin_cpu_params": true, + "pipeline_model_parallel_comm_backend": null, + "pipeline_model_parallel_layout": null, + "pipeline_model_parallel_size": 1, + "position_embedding_type": "rope", + "pretrained_checkpoint": null, + "profile": false, + "profile_ranks": "[]", + "profile_step_end": 12, + "profile_step_start": 10, + "pytorch_profiler_collect_callstack": false, + "pytorch_profiler_collect_chakra": false, + "pytorch_profiler_collect_shapes": false, + "q_lora_rank": null, + "qk_clip": false, + "qk_clip_alpha": 0.5, + "qk_clip_threshold": 100, + "qk_head_dim": 128, + "qk_l2_norm": false, + "qk_layernorm": false, + "qk_pos_emb_head_dim": 64, + "query_in_block_prob": 0.1, + "quick_geglu": false, + "rampup_batch_size": null, + "rank": 0, + "recipe_optimizer": "{'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}", + "recipe_overrides": "{}", + "recipe_requested_batch_tokens": 262144, + "recipe_schedule": "{'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}", + "recompute_granularity": null, + "recompute_method": null, + "recompute_modules": null, + "recompute_num_layers": null, + "record_memory_history": false, + "record_shapes": false, + "refit_execution_batch_bytes": null, + "refit_method": "gloo", + "relative_attention_max_distance": 128, + "relative_attention_num_buckets": 32, + "replication": false, + "replication_factor": 2, + "replication_jump": null, + "rerun_mode": "validate_results", + "reset_attention_mask": false, + "reset_position_ids": false, + "result_rejected_tracker_filename": null, + "retriever_report_topk_accuracies": "[]", + "retriever_score_scaling": false, + "reuse_grad_buf_for_mxfp8_param_ag": false, + "rl_consumption_granularity": "B", + "rl_default_temperature": 1.0, + "rl_default_top_k": -1, + "rl_default_top_p": 0, + "rl_durable_rollout_bank": false, + "rl_generation_lag": null, + "rl_importance_sampling_truncation_coef": null, + "rl_inference_expert_model_parallel_size": null, + "rl_inference_expert_tensor_model_parallel_size": null, + "rl_inference_logprobs_is_correction": false, + "rl_inference_model_unified_memory_level": 0, + "rl_inference_parsers": "[]", + "rl_inference_pipeline_model_parallel_size": null, + "rl_inference_tensor_model_parallel_size": null, + "rl_kv_cache_management_mode": "persist", + "rl_max_inflight_requests": null, + "rl_offload_inference_model_weights_when_idle": false, + "rl_offload_optimizer_during_inference": false, + "rl_partial_rollouts": false, + "rl_persist_cuda_graphs": false, + "rl_profile": false, + "rl_profile_dir": null, + "rl_prompts_per_eval": 32, + "rl_rollout_bank_dir": null, + "rl_rollout_bank_max_bytes": 0, + "rl_sequence_packing_algo": "fifo", + "rl_sequence_packing_max_sequences_per_bin": 50, + "rl_skip_bos_token": false, + "rl_submission_granularity": "B", + "rl_training_cuda_graphs": false, + "rl_use_sequence_packing": false, + "rl_verify_model_weights_swap": false, + "rope_scaling_factor": 8.0, + "rope_type": null, + "rotary_base": 10000, + "rotary_interleaved": false, + "rotary_percent": 1.0, + "rotary_scaling_factor": 1.0, + "rotary_seq_len_interpolation_factor": null, + "run_workload_inspector_server": false, + "sample_rate": 1.0, + "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s12000_to26703/checkpoints", + "save_activations_interval": null, + "save_dgrads_interval": null, + "save_interval": 4000, + "save_params_interval": null, + "save_retain_interval": null, + "save_tokenizer_assets": true, + "save_tokens_per_expert_interval": null, + "save_wgrads_interval": null, + "seed": 42, + "seq_length": 2048, + "sequence_packing_scheduler": null, + "sequence_parallel": false, + "sft": false, + "sft_mock_dataset_config_json": null, + "sft_tokenizer_prompt_format": "nemotron-h-aligned", + "sgd_momentum": 0.9, + "sharp_enabled_group": null, + "short_seq_prob": 0.1, + "skip_train": false, + "skipped_train_samples": 0, + "softmax_type": "vanilla", + "spec": null, + "special_tokens": null, + "split": null, + "squared_relu": false, + "start_eval_at_iter": null, + "start_weight_decay": 0.0, + "step_batch_size_schedule": null, + "straggler_ctrlr_port": 65535, + "straggler_minmax_count": 1, + "strict_fsdp_dtensor_load": true, + "suggested_communication_unit_size": null, + "swiglu": false, + "swin_backbone_type": "tiny", + "symmetric_ar_type": null, + "te_precision_config_file": null, + "te_rng_tracker": false, + "tensor_model_parallel_size": 1, + "tensor_parallel_num_weight_shards": 1, + "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s12000_to26703/tensorboard", + "tensorboard_log_interval": 1, + "tensorboard_queue_size": 1000, + "test_data_path": null, + "test_mode": false, + "tiktoken_num_special_tokens": 1000, + "tiktoken_pattern": null, + "timing_log_level": 0, + "timing_log_option": "minmax", + "titles_data_path": null, + "tokenizer_hf_no_include_special_tokens": false, + "tokenizer_hf_no_use_fast": false, + "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "tokenizer_sentencepiece_ignore_extra_whitespaces": true, + "tokenizer_sentencepiece_legacy": false, + "tokenizer_type": "HuggingFaceTokenizer", + "torch_fsdp2_reshard_after_forward": true, + "tp_comm_bootstrap_backend": "nccl", + "tp_comm_bulk_dgrad": true, + "tp_comm_bulk_wgrad": true, + "tp_comm_overlap": false, + "tp_comm_overlap_ag": true, + "tp_comm_overlap_cfg": null, + "tp_comm_overlap_rs": true, + "tp_comm_overlap_rs_dgrad": false, + "tp_comm_split_ag": true, + "tp_comm_split_rs": true, + "train_data_path": null, + "train_iters": 26703, + "train_samples": null, + "train_sync_interval": null, + "training_recipe": "paper2604_21106", + "training_recipe_metadata": "{'recipe': 'paper2604_21106', 'width': 1408, 'total_tokens': 7000031232, 'batch_tokens': 262144, 'reference_width': 640, 'reference_tokens': 983315200, 'reference_batch_tokens': 262144, 'base_lrs': {'matrix': 0.014, 'embedding': 0.3, 'unembedding': 0.004, 'norm': 0.005, 'other': 0.005}, 'peak_lrs': {'matrix': 0.007470638533717786, 'embedding': 0.2022599587389726, 'unembedding': 0.0026967994498529684, 'norm': 0.00337099931231621, 'other': 0.00337099931231621}, 'reference': 'paper vanilla 0-20-0 at width 640; configurable token anchor', 'tied_embedding_policy': 'unembedding (one AdamW update)', 'other_parameter_policy': 'norm LR (extension for non-paper scalars/biases)', 'weight_decay': 0.0, 'warmup_steps': 0, 'final_lr_fraction': 0.1, 'optimizer': {'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}, 'schedule': {'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}, 'group_weight_decay': {'matrix': 0.0, 'embedding': 0.0, 'unembedding': 0.0, 'norm': 0.0, 'other': 0.0}, 'recipe_overrides': {}, 'requested_batch_tokens': 262144, 'sequence_length': 2048, 'batch_assumptions': {'global_batch_tokens': 262144, 'sequence_length': 2048, 'round_up': False}, 'lr_transfer': 'mup_hyperp_released_source', 'assumptions': [], 'backend_kernel_assumptions': {}, 'paper_settings_overridden': False, 'model': {'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}}", + "transformer_impl": "local", + "transformer_pipeline_model_parallel_size": 1, + "trust_remote_code": false, + "untie_embeddings_and_output_weights": true, + "use_checkpoint_args": false, + "use_checkpoint_opt_param_scheduler": false, + "use_cpu_initialization": null, + "use_dist_ckpt": false, + "use_dist_ckpt_deprecated": false, + "use_distributed_optimizer": false, + "use_flash_attn": false, + "use_fused_mhc": false, + "use_fused_weighted_squared_relu": false, + "use_gigatoken": false, + "use_gloo_process_groups": true, + "use_grouped_gemm_for_dense_mlp": false, + "use_grouped_gemm_for_shared_expert": false, + "use_kitchen_attention": false, + "use_layer_wise_distributed_optimizer": false, + "use_layer_wise_param_layout": true, + "use_legacy_static_engine": false, + "use_mamba_mem_eff_path": true, + "use_megatron_fsdp": false, + "use_mp_args_from_checkpoint_args": false, + "use_mup": false, + "use_one_sent_docs": false, + "use_persistent_ckpt_worker": false, + "use_precision_aware_optimizer": false, + "use_pytorch_profiler": false, + "use_ring_exchange_p2p": false, + "use_rope_scaling": false, + "use_rotary_position_embeddings": false, + "use_sharp": false, + "use_te_activation_func": false, + "use_tokenizer_model_from_checkpoint_args": true, + "use_torch_fsdp2": false, + "use_torch_optimizer_for_cpu_offload": false, + "use_tp_pp_dp_mapping": false, + "use_transformer_engine_op_fuser": false, + "use_varlen_dataset": false, + "v_head_dim": 128, + "valid_data_path": null, + "validation_set_names": null, + "variable_seq_lengths": false, + "varlen_mock_dataset_config_json": null, + "varlen_sbhd_validation": false, + "verify_integrity": false, + "virtual_pipeline_model_parallel_size": null, + "vision_backbone_type": "vit", + "vision_pretraining": false, + "vision_pretraining_type": "classify", + "vocab_extra_ids": 0, + "vocab_file": null, + "vocab_size": 32001, + "wandb_entity": "LoopTF-4-CSPs", + "wandb_exp_name": "r4_la_hf_warmstart_s12000_to26703", + "wandb_project": "loopTF-workspace", + "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s12000_to26703/wandb", + "weight_decay": 0.0, + "weight_decay_incr_style": "constant", + "wgrad_deferral_limit": 0, + "window_attn_skip_freq": null, + "window_size": null, + "world_size": 16, + "yaml_cfg": null, + "yarn_beta_fast": null, + "yarn_beta_slow": null, + "yarn_correction_range_round_to_int": null, + "yarn_original_max_position_embeddings": null + }, + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3967744, + "lookahead_parameters": 93191296, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659335424, + "total_trainable": 659335424 + }, + "pipeline_model_parallel_size": 1, + "run_name": "r4_weights_only_d0_p0_a2_r0_s12000_to26703", + "source_run_directory": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s12000_to26703", + "tensor_model_parallel_size": 1, + "tokenizer": { + "added_tokens": [ + "<|assistant_end|>" + ], + "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training_budget_tokens": 7000031232, + "training_code_revision": "8219ffa08d51818b9b6b150cbee1b35089dc91af", + "training_tokens": 7000031232, + "warm_start_step": 12000 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..e10dbbab8bd39a8b3003247b7e2b55e5886c6447 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/launch_config.env @@ -0,0 +1,14 @@ +# HF safetensors warm start: copied FBW weights, fresh optimizer/RNG/data stream. +source "$(dirname "${BASH_SOURCE[0]}")/lookahead.env" +RUN_NAME=r4_weights_only_d0_p0_a2_r0 +# Continue without activation checkpointing in either the copied backbone or +# the freshly initialized refiner blocks. +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +# The generated native checkpoint carries a fresh optimizer/RNG whose scheduler +# is advanced to the source step. Online-loader state intentionally starts fresh. +ONLINE_SOURCE_MODE=stream +# Reuse the immutable held-out set already qualified by successful multi-node +# Look-ahead runs; streaming validation can leave ranks waiting at different rates. +ONLINE_VALIDATION_MANIFEST="${PROJECT_ROOT}/data/validation/manual_14b/manifest.json" +WANDB_TAGS=product,resume,R4,transition,lookahead,D0,P0,A2,R0,fbw_norms,weights_only,20l,7b diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..8a2ea07f0823d8ee313aec14a6fb56a95d583b9e --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "sha256": "af24c64ebe15adbd9fb366f11932d03f41a650f2d35d4ff2eeaf73e00bac75bf", + "size": 747 + }, + "config.json": { + "sha256": "4666cc510a935f8dc9a7dcc2331e2c3e75291c1af459ab54e2888d868b32ca55", + "size": 46454 + }, + "launch_config.env": { + "sha256": "74721ad10af3e65776cd7e5270239a979f4ef83b4ef8583eaee6ec30fc49dfd3", + "size": 878 + }, + "model.safetensors": { + "sha256": "40211d8c44e1ffcede0c9c791831e20ec54230fd29b51820e75d7776e3176e61", + "size": 1318836848 + }, + "training_recipe.json": { + "sha256": "9ed332689297ca4fbeaeaf5d5f0be3a9f4503792fb6390032e3e650b712c945a", + "size": 4472 + } + }, + "note": "manifest.json does not hash itself" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aaf9e987e31e4dbc604f0f3af1f0b04e895792fe --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40211d8c44e1ffcede0c9c791831e20ec54230fd29b51820e75d7776e3176e61 +size 1318836848 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..afe10f9fee24a774d3a95588c119129972cf1e42 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart12000-to26703--my34axon/step_26703/training_recipe.json @@ -0,0 +1,153 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659335424, + "feedback_parameters": 3967744, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93191296, + "total_parameters": 659335424 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..9216c5b164f76ff84daee3eaec8aa8845539898b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "7t0tefok", + "previous_wandb_name": "r4_la_hf_warmstart_s16000_to26703", + "training_run_name": "r4_la_hf_warmstart_s16000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s16000_to26703" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..700fabeb45f65b956c25c408fcd518f8e74f79ed --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/README.md @@ -0,0 +1,11 @@ +# R4 Look-ahead final weights (16000 warm start) + +This is the final step 26,703 weights-only export for `r4_weights_only_d0_p0_a2_r0_s16000_to26703`, warm-started from the FBW export at step 16000. + +## Format + +`model.safetensors` contains the native Look-ahead Megatron model state only. It excludes optimizer, scheduler, RNG, dataloader, caches, logs, and credentials. It is not a Transformers `AutoModel.from_pretrained` checkpoint. + +To load, use the code revision in `config.json`, construct `lookahead_tf.hf.model.LookaheadTransformer` with `model_config` and `lookahead_config`, remove exactly the `model.` prefix from each tensor key, and call `load_state_dict(..., strict=True, assign=True)`. + +Use `manifest.json` to verify payload hashes. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..aeb53ee494d5be961979e9a1507dcf6bd0617db1 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/config.json @@ -0,0 +1,1041 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "excluded_state": [ + "optimizer", + "opt_param_scheduler", + "rng_state", + "dataloader" + ], + "iteration": 26703, + "loading": { + "format": "native PyTorch state dict saved with safetensors; not a Transformers AutoModel", + "instructions": "Construct LookaheadTransformer(model_config, lookahead_config), remove exactly the model. prefix, then load_state_dict(..., strict=True, assign=True).", + "key_prefix": "model." + }, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 0.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "model_arguments": { + "_is_global_batch_size_explicitly_specified": true, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "accumulate_allreduce_grads_in_fp32": true, + "activation_func_clamp_value": null, + "activation_offload_fraction": 1.0, + "adam_beta1": 0.8, + "adam_beta2": 0.95, + "adam_eps": 1e-10, + "adaptive_muon_beta2": 0.95, + "adaptive_muon_eps": 1e-10, + "adaptive_muon_moment2_method": "normuon", + "add_bias_linear": false, + "add_position_embedding": true, + "add_qkv_bias": false, + "adlr_autoresume": false, + "adlr_autoresume_interval": 1000, + "align_grad_reduce": true, + "align_param_gather": false, + "allow_ambiguous_pad_tokens": false, + "app_tag_run_name": null, + "app_tag_run_version": "0.0.0", + "apply_query_key_layer_scaling": false, + "apply_residual_connection_post_layernorm": false, + "apply_rope_fusion": false, + "apply_wd_to_qk_layernorm": false, + "architecture": "lookahead", + "architecture_module": "[]", + "async_ckpt_cpu_priority": 10, + "async_ckpt_io_priority": 3, + "async_ckpt_use_cpu_shm": false, + "async_save": false, + "async_strategy": "mcore", + "attention_backend": "AttnBackend.auto", + "attention_cp_layout": "zigzag", + "attention_dropout": 0.0, + "attention_output_gate": false, + "attention_softmax_in_fp32": false, + "auto_detect_ckpt_format": false, + "backbone_metadata": "{'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}", + "backbone_profile": "paper2604_vanilla", + "barrier_with_L1_time": true, + "batch_invariant_backend": "te_native", + "batch_invariant_collective": "ordered", + "batch_invariant_mode": false, + "bert_binary_head": true, + "bert_embedder_type": "megatron", + "bert_load": null, + "bf16": true, + "bias_dropout_fusion": false, + "bias_gelu_fusion": false, + "bias_swiglu_fusion": true, + "biencoder_projection_dim": 0, + "biencoder_shared_query_context_model": false, + "block_data_path": null, + "cache_mla_latents": false, + "calc_ft_timeouts": false, + "calculate_per_token_loss": false, + "chat_template": null, + "check_for_large_grads": false, + "check_for_nan_in_loss_and_grad": true, + "check_for_spiky_loss": false, + "check_weight_hash_across_dp_replicas_interval": null, + "ckpt_assume_constant_structure": false, + "ckpt_convert_format": null, + "ckpt_convert_save": null, + "ckpt_convert_update_legacy_dist_opt_format": false, + "ckpt_drop_redundant_extra_state": false, + "ckpt_format": "torch", + "ckpt_fully_parallel_load": false, + "ckpt_fully_parallel_load_exchange_algo": "broadcast", + "ckpt_fully_parallel_load_per_rank_objects": false, + "ckpt_fully_parallel_load_process_group": "dp", + "ckpt_fully_parallel_save": true, + "ckpt_fully_parallel_save_deprecated": false, + "ckpt_fully_parallel_save_process_group": "dp", + "ckpt_load_validate_sharding_integrity": true, + "ckpt_pg_tensors_cache_create": false, + "ckpt_pg_tensors_cache_path": null, + "ckpt_step": null, + "classes_fraction": 1.0, + "clip_grad": 0.0, + "clone_scatter_output_in_embedding": true, + "config_logger_dir": "", + "consumed_train_samples": 3417984, + "consumed_valid_samples": 68352, + "context_parallel_size": 1, + "cp_comm_type": "['p2p']", + "cpu_offloading_num_layers": 0, + "cpu_offloading_retain_pinned_cpu_buffers": false, + "create_all_gather_group": false, + "create_attention_mask_in_dataloader": false, + "cross_entropy_fusion_impl": "native", + "cross_entropy_loss_fusion": false, + "csa_compress_ratios": null, + "csa_compress_rotary_base": 40000.0, + "csa_dense_mode": false, + "csa_window_size": 128, + "cuda_graph_impl": "none", + "cuda_graph_modules": "[]", + "cuda_graph_warmup_steps": 3, + "curr_iteration": 26702, + "data_args_path": null, + "data_cache_path": null, + "data_parallel_random_init": false, + "data_parallel_sharding_strategy": "optim_grads_params", + "data_parallel_size": 16, + "data_path": null, + "data_per_class_fraction": 1.0, + "data_sharding": true, + "dataloader_defer_npy_index_mmap": false, + "dataloader_fast_cache_load": false, + "dataloader_inter_document_masking": false, + "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s16000_to26703/dataloader", + "dataloader_type": "external", + "ddp_average_in_collective": false, + "ddp_bucket_size": null, + "ddp_num_buckets": null, + "ddp_pad_buckets_for_high_nccl_busbw": false, + "ddp_param_name_patterns_for_fp32_local_accumulation": "[]", + "ddp_reduce_scatter_with_fp32_accumulation": false, + "decode_only_cuda_graphs": false, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "decoder_num_layers": null, + "decoder_seq_length": null, + "decoupled_lr": null, + "decoupled_min_lr": null, + "decrease_batch_size_if_needed": false, + "defer_embedding_wgrad_compute": false, + "delay_offload_until_cuda_graph": false, + "delay_wgrad_compute": false, + "delta_offload_bytes_across_pp_ranks": 0, + "deprecated_use_mcore_models": false, + "deterministic_mode": false, + "dino_bottleneck_size": 256, + "dino_freeze_last_layer": 1, + "dino_head_hidden_size": 2048, + "dino_local_crops_number": 10, + "dino_local_img_size": 96, + "dino_norm_last_layer": false, + "dino_teacher_temp": 0.07, + "dino_warmup_teacher_temp": 0.04, + "dino_warmup_teacher_temp_epochs": 30, + "disable_bf16_reduced_precision_matmul": false, + "disable_jit_fuser": false, + "disable_msc_deprecated": false, + "disable_straggler_on_startup": false, + "disable_symmetric_registration": false, + "dist_ckpt_format_deprecated": null, + "dist_ckpt_optim_fully_reshardable": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_strictness": "assume_ok_unexpected", + "dist_ckpt_workers": 1, + "distrib_optim_fully_reshardable_mem_efficient": false, + "distribute_saved_activations": false, + "distributed_backend": "nccl", + "distributed_timeout_minutes": 10, + "distributed_timeout_seconds_after_init": null, + "do_test": 0, + "do_train": 1, + "do_valid": 1, + "dsa_indexer_head_dim": null, + "dsa_indexer_k_norm_epsilon": null, + "dsa_indexer_k_norm_fp32": false, + "dsa_indexer_loss_coeff": null, + "dsa_indexer_n_heads": null, + "dsa_indexer_rope_interleaved": false, + "dsa_indexer_rotate_activation": true, + "dsa_indexer_scoring_relu": true, + "dsa_indexer_skip_topk_offset": 0, + "dsa_indexer_topk": null, + "dsa_indexer_topk_freq": 1, + "dsa_indexer_use_sparse_loss": false, + "dsa_kernel_backend": "none", + "dump_param_to_param_group_map": null, + "embedding_init_method_std": null, + "embedding_path": null, + "empty_unused_memory_level": 0, + "enable_chunked_prefill": false, + "enable_cuda_graph": false, + "enable_experimental": false, + "enable_ft_package": false, + "enable_full_sharding_in_hsdp": false, + "enable_mhc_connections": false, + "enable_msc": false, + "enable_one_logger": true, + "encoder_num_layers": 20, + "encoder_seq_length": 2048, + "end_weight_decay": 0.0, + "eod_mask_loss": false, + "ep_overlap_early_attn_memory_release": false, + "error_injection_rate": 0, + "error_injection_type": "transient_error", + "eval_at_start": true, + "eval_global_batch_size": 128, + "eval_interval": 100, + "eval_iters": 2, + "eval_micro_batch_size": 8, + "evaluation_metadata": "{'eval_at_start': True, 'initial_iters': 2, 'intermediate_iters': 2, 'final_validation_iters': 270, 'test_iters': 0, 'global_batch_size': 128, 'sequence_length': 2048, 'intermediate_input_tokens': 524288, 'final_validation_sequences': 34560, 'final_validation_input_tokens': 70778880, 'validation_iterator': 'continues_existing_held_out_iterator'}", + "evidence_data_path": null, + "exit_duration_in_mins": null, + "exit_interval": 26703, + "exit_on_missing_checkpoint": true, + "exit_signal": 15, + "exit_signal_handler": false, + "exit_signal_handler_for_dataloader": false, + "exit_signal_handler_for_training": false, + "exp_avg_dtype": "torch.float32", + "exp_avg_sq_dtype": "torch.float32", + "experimental_attention_variant": null, + "expert_data_parallel_sharding_strategy": null, + "expert_gtp_weight_remat_size": 1, + "expert_model_parallel_size": 1, + "expert_tensor_parallel_num_weight_shards": 1, + "expert_tensor_parallel_size": 1, + "external_cuda_graph": false, + "fake_process_group": false, + "fault_injector_delay_start_iteration": null, + "fault_injector_fault_delay": null, + "fault_injector_fault_probabilities": null, + "fault_injector_fault_types": null, + "fault_injector_mtti_seconds": null, + "fault_injector_num_ranks": null, + "fault_injector_offset_seconds": null, + "fault_injector_ranks": null, + "fault_injector_seed": null, + "feedback_normuon_beta2": 0.95, + "feedback_scalar_lr": 0.0005, + "feedback_scalar_weight_decay": 0.0, + "feedback_z_loss_coefficient": 0.0, + "feedback_z_loss_start_fraction": 1.0, + "ffn_hidden_size": 5632, + "fim_data": false, + "fim_eod_token": "<|endoftext|>", + "fim_fragment_rate": null, + "fim_middle_token": "", + "fim_no_prefix": null, + "fim_pad_token": "", + "fim_prefix_token": "", + "fim_rate": 0.5, + "fim_split_sample": null, + "fim_spm_rate": 0.5, + "fim_suffix_token": "", + "final_eval_iters": 270, + "fine_grained_activation_offloading": false, + "fine_grained_offloading_max_inflight_offloads": null, + "finetune": false, + "first_last_layers_bf16": false, + "flash_attention_version": null, + "flash_decode": false, + "flight_recorder_dump_on_timeout": true, + "flight_recorder_dump_path": null, + "flight_recorder_extra_dump_on_exec": true, + "flight_recorder_include_only_active": true, + "flight_recorder_include_stack_trace": false, + "flight_recorder_trace_buffer_size": 2000, + "fp16": false, + "fp16_lm_cross_entropy": false, + "fp32_residual_connection": false, + "fp4": null, + "fp4_param_gather": false, + "fp4_quantizer_factory": null, + "fp4_recipe": "nvfp4", + "fp8": null, + "fp8_amax_compute_algo": "most_recent", + "fp8_amax_history_len": 1, + "fp8_interval": 1, + "fp8_margin": 0, + "fp8_output_proj": false, + "fp8_param_gather": false, + "fp8_quantizer_factory": null, + "fp8_recipe": "delayed", + "fp8_wgrad": true, + "freeze_all_layers": false, + "fsdp_db_use_persist_buf_on_alloc_fail": false, + "fsdp_double_buffer": false, + "fsdp_manual_registration": false, + "ft_num_warmup_iters": 5, + "full_validation": false, + "fused_residual_rmsnorm": false, + "gdp_cutedsl_kernel": false, + "gdp_num_chunk_states_to_recompute": 2, + "gdp_num_householder": 3, + "global_batch_size": 128, + "glu_linear_offset": 0.0, + "gpu_sniff_test_interval": null, + "grad_reduce_in_bf16": false, + "gradient_accumulation_fusion": false, + "gradient_reduce_div_fusion": true, + "group_query_attention": false, + "grpo_clamp_eps_lower": 0.01, + "grpo_clamp_eps_upper": 0.01, + "grpo_entropy_term_weight": 0.0, + "grpo_filter_groups_with_same_reward": false, + "grpo_group_size": 2, + "grpo_iterations": 2, + "grpo_kl_beta": 0.001, + "grpo_prompts_per_step": 32, + "gtp_expert_remat_nccl_ub": false, + "gtp_remat_nccl_ub": false, + "gtp_remat_opt_in_modules": "[]", + "gtp_remat_reduce_scatter_with_fp32_accumulation": false, + "gtp_weight_remat_size": 1, + "head_lr_mult": 1.0, + "heterogeneous_layers_config_encoded_json": null, + "heterogeneous_layers_config_path": null, + "hfsdp_param_gather_overlap": false, + "hidden_dropout": 0.0, + "hidden_size": 1408, + "hierarchical_context_parallel_sizes": null, + "high_priority_a2a_comm_stream": false, + "high_priority_stream_groups": "[]", + "hybrid_context_parallel": false, + "hybrid_layer_pattern": null, + "hybrid_override_pattern": null, + "hysteresis": 2, + "ict_head_size": null, + "ict_load": null, + "img_h": 224, + "img_w": 224, + "indexer_batch_size": 128, + "indexer_log_interval": 1000, + "inference_batch_times_seqlen_threshold": -1, + "inference_coordinator_port": null, + "inference_cuda_graph_all_prefills": false, + "inference_cuda_graph_max_tokens": 512, + "inference_cuda_graph_scope": "InferenceCudaGraphScope.none", + "inference_disable_ep_consensus": false, + "inference_disable_triton_nvls_kernels": false, + "inference_dynamic_batching": false, + "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, + "inference_dynamic_batching_async_sched_mode": "async", + "inference_dynamic_batching_block_size": 256, + "inference_dynamic_batching_buffer_size_gb": 40.0, + "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, + "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", + "inference_dynamic_batching_enable_prefix_caching": false, + "inference_dynamic_batching_logprobs_mode": "raw_logprobs", + "inference_dynamic_batching_mamba_memory_ratio": null, + "inference_dynamic_batching_max_requests": null, + "inference_dynamic_batching_max_tokens": null, + "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", + "inference_dynamic_batching_media_cache_routing_weight": 1.0, + "inference_dynamic_batching_num_cuda_graphs": 16, + "inference_dynamic_batching_paused_buffer_size_gb": null, + "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, + "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", + "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", + "inference_dynamic_batching_prefix_caching_mamba_gb": null, + "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, + "inference_dynamic_batching_sampling_backend": "torch", + "inference_dynamic_batching_track_generated_token_events": false, + "inference_dynamic_batching_track_paused_request_events": false, + "inference_dynamic_batching_unified_memory_level": 0, + "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, + "inference_flashinfer_mxfp8_token_capacity": null, + "inference_fuse_tp_communication": false, + "inference_grouped_gemm_backend": "vllm", + "inference_logging_step_interval": 0, + "inference_max_requests": 8, + "inference_max_seq_length": 2560, + "inference_moe_disable_fused_quant_kernels": false, + "inference_moe_token_dispatcher_type": "nvls", + "inference_rng_tracker": false, + "inference_shards": null, + "inference_text_gen_server_logging": false, + "inference_use_synchronous_zmq_collectives": false, + "inference_wandb_logging": false, + "init_method_std": 0.02, + "init_method_xavier_uniform": false, + "init_model_with_meta_device": false, + "initial_loss_scale": 4294967296, + "inprocess_active_world_size": 16, + "inprocess_barrier_timeout": 120, + "inprocess_completion_timeout": 120, + "inprocess_empty_cuda_cache": false, + "inprocess_granularity": "node", + "inprocess_hard_timeout": 90, + "inprocess_heartbeat_interval": 30, + "inprocess_heartbeat_timeout": 60, + "inprocess_last_call_wait": 1, + "inprocess_max_iterations": null, + "inprocess_monitor_process_interval": 1.0, + "inprocess_monitor_thread_interval": 1.0, + "inprocess_progress_watchdog_interval": 1.0, + "inprocess_restart": false, + "inprocess_soft_timeout": 60, + "inprocess_termination_grace_time": 1, + "is_hybrid_model": false, + "iter_per_epoch": 1250, + "iteration": 16000, + "iterations_to_skip": "[]", + "keep_fp8_transpose_cache": false, + "kitchen_attention_backend": "sdpa", + "kv_channels": 128, + "kv_lora_rank": 32, + "langrl_env_config": null, + "layernorm_epsilon": 1e-06, + "layernorm_zero_centered_gamma": false, + "lazy_mpu_init": false, + "linear_attention_freq": null, + "linear_conv_kernel_dim": 4, + "linear_cp_layout": "zigzag", + "linear_cp_mode": "headwise", + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "lion_beta1": 0.95, + "lion_beta2": 0.98, + "load": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/transfers/hf_weights_only/s16000/lookahead/checkpoints", + "load_main_params_from_ckpt": false, + "local_rank": 0, + "log_device_memory_used": false, + "log_energy": false, + "log_interval": 1, + "log_loss_scale_to_tensorboard": true, + "log_max_attention_logit": false, + "log_memory_interval": null, + "log_memory_to_tensorboard": false, + "log_num_zeros_in_grad": false, + "log_params_norm": false, + "log_progress": false, + "log_straggler": false, + "log_throughput": false, + "log_timers_to_tensorboard": true, + "log_validation_ppl_to_tensorboard": false, + "log_world_size_to_tensorboard": false, + "logging_level": null, + "logit_dtype": null, + "logits_load_decode_threads": 4, + "logits_load_dir": null, + "logits_load_ignore_errors": false, + "logits_load_kd_loss_alpha": 1.0, + "logits_load_msc_prefetch_depth": 2, + "logits_load_prefetch_factor": 3, + "logits_save_dir": null, + "logits_save_dtype": "fp16", + "logits_save_top_k": null, + "logits_save_top_p": null, + "logits_save_top_p_min_k": 1, + "lookahead_allow_objective_change_on_resume": false, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_cache_layers": null, + "lookahead_checkpoint_refiner": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_full_attention_interval": 6, + "lookahead_future_gradient_mode": "all", + "lookahead_future_loss_weight": 0.0, + "lookahead_gradient_checkpointing": false, + "lookahead_mode": "parallel", + "lookahead_normalize_feedback_embedding": true, + "lookahead_normalize_feedback_output": true, + "lookahead_num_passes": 3, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_temperature": 1.0, + "lookahead_pseudo_top_k": 128, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_z_loss_coefficient": 0.0, + "loss_scale": null, + "loss_scale_window": 1000, + "lr": 0.007470638533717786, + "lr_decay_iters": null, + "lr_decay_samples": null, + "lr_decay_style": "linear", + "lr_warmup_fraction": null, + "lr_warmup_init": 0.0, + "lr_warmup_iters": 0, + "lr_warmup_samples": 0, + "lr_wsd_decay_iters": 0, + "lr_wsd_decay_samples": null, + "lr_wsd_decay_style": "linear", + "main_grads_dtype": "torch.float32", + "main_params_dtype": "torch.float32", + "make_vocab_size_divisible_by": 64, + "mamba_head_dim": 64, + "mamba_inference_conv_states_dtype": "torch.bfloat16", + "mamba_inference_ssm_states_dtype": "torch.bfloat16", + "mamba_num_groups": 8, + "mamba_num_heads": null, + "mamba_state_dim": 128, + "mamba_training_ssm_states_dtype": null, + "manual_gc": false, + "manual_gc_eval": true, + "manual_gc_interval": 0, + "mask_factor": 1.0, + "mask_prob": 0.15, + "mask_type": "random", + "masked_softmax_fusion": true, + "max_position_embeddings": 2048, + "max_seqlen_per_dp_cp_rank": null, + "max_tokens_to_oom": 12000, + "megatron_fsdp_enable_fine_grained_param_gather": false, + "megatron_fsdp_grad_comm_dtype": null, + "megatron_fsdp_main_grads_dtype": null, + "megatron_fsdp_main_params_dtype": "torch.float32", + "megatron_fsdp_max_pool_double_buffer": false, + "megatron_fsdp_version": 1, + "megatron_lm_path": "/root/dl/runtime/Megatron-LM", + "memory_snapshot_path": "snapshot.pickle", + "merge_file": null, + "metadata_path": null, + "mhc_fused_backend": "auto", + "mhc_init_gating_factor": 0.01, + "mhc_num_residual_streams": 4, + "mhc_recompute_layer_num": null, + "mhc_sinkhorn_iterations": 20, + "micro_batch_size": 8, + "microbatch_group_size_per_vp_stage": null, + "mid_level_dataset_surplus": 0.005, + "min_loss_scale": 1.0, + "min_lr": 0.0007470638533717787, + "min_offloaded_tensor_size": 1048576, + "mla_down_proj_fusion": false, + "mlp_chunks_for_prefill": 1, + "mlp_chunks_for_training": 1, + "mmap_bin_files": true, + "mock_data": false, + "moe_apply_probs_on_input": false, + "moe_aux_loss_coeff": 0.0, + "moe_combine_bwd_dtype": "bf16", + "moe_deepep_num_sms": null, + "moe_dispatch_fwd_dtype": "bf16", + "moe_enable_deepep": false, + "moe_enable_routing_replay": false, + "moe_expert_capacity_factor": null, + "moe_expert_rank_capacity_factor": null, + "moe_ffn_hidden_size": null, + "moe_flex_dispatcher_backend": "deepep", + "moe_flex_dispatcher_num_sms": null, + "moe_grouped_gemm": false, + "moe_hybridep_num_blocks_permute": null, + "moe_hybridep_num_blocks_unpermute": null, + "moe_hybridep_num_sms": null, + "moe_hybridep_num_sms_preprocessing": 108, + "moe_hybridep_pad_uneven_dispatch_inputs": false, + "moe_input_jitter_eps": null, + "moe_latent_size": null, + "moe_layer_freq": 1, + "moe_layer_recompute": false, + "moe_mlp_glu_interleave_size": null, + "moe_ncclep_zero_copy": false, + "moe_pad_expert_input_to_capacity": false, + "moe_pad_experts_for_cuda_graph_inference": false, + "moe_paged_stash": false, + "moe_paged_stash_buffer_size_factor_cpu": 0.0, + "moe_paged_stash_buffer_size_factor_cuda": 1.1, + "moe_paged_stash_page_size": 64, + "moe_per_layer_logging": false, + "moe_permute_fusion": false, + "moe_permute_fusion_into_hybridep": false, + "moe_router_bias_update_rate": 0.001, + "moe_router_dtype": null, + "moe_router_enable_expert_bias": false, + "moe_router_force_biased": null, + "moe_router_force_load_balancing": false, + "moe_router_fusion": false, + "moe_router_group_topk": null, + "moe_router_load_balancing_type": "aux_loss", + "moe_router_num_groups": null, + "moe_router_padding_for_fp8": false, + "moe_router_padding_for_quantization": false, + "moe_router_pre_softmax": false, + "moe_router_quantile_balancing_ema": 0.0, + "moe_router_score_function": "softmax", + "moe_router_skip_muon": false, + "moe_router_topk": 2, + "moe_router_topk_scaling_factor": null, + "moe_routing_trace_capture_hidden_states": false, + "moe_routing_trace_capture_logits": false, + "moe_routing_trace_dump_weights": false, + "moe_routing_trace_max_training_iters": null, + "moe_routing_trace_path": null, + "moe_shared_expert_gate": false, + "moe_shared_expert_glu_interleave_size": null, + "moe_shared_expert_intermediate_size": null, + "moe_shared_expert_overlap": false, + "moe_single_grouped_bias": false, + "moe_single_grouped_weight": false, + "moe_token_dispatcher_type": "allgather", + "moe_token_drop_policy": "probs", + "moe_upcycling_granularity": 1, + "moe_use_grouped_tensor": false, + "moe_use_norm_before_up_proj": false, + "moe_use_upcycling": false, + "moe_z_loss_coeff": null, + "mrope_section": null, + "mscale": 1.0, + "mscale_all_dim": 0.0, + "mtp_detach_heads": false, + "mtp_hsm": false, + "mtp_hybrid_override_pattern": null, + "mtp_loss_scaling_factor": 0.1, + "mtp_num_layers": null, + "mtp_standalone": false, + "mtp_use_repeated_layer": false, + "multi_latent_attention": false, + "multiple_validation_sets": false, + "muon_coefficient_type": "quintic", + "muon_extra_scale_factor": 1.0, + "muon_fp32_matmul_prec": "medium", + "muon_momentum": 0.95, + "muon_nesterov": true, + "muon_num_ns_steps": 5, + "muon_scalar_optimizer": "adam", + "muon_scale_mode": "spectral", + "muon_split_qkv": true, + "muon_tp_mode": "duplicated", + "muon_use_syrk": false, + "mup_attn_scale_power": 1.0, + "mup_base_head_dim": null, + "mup_base_hidden_size": null, + "mup_embedding_mult": 1.0, + "mup_output_mult": 1.0, + "mup_width_mult": 1.0, + "nccl_all_reduce_for_prefill": false, + "nccl_communicator_config_path": null, + "nccl_ub": false, + "no_load_optim": null, + "no_load_rng": null, + "no_persist_layer_norm": false, + "no_rope_freq": null, + "no_save_optim": null, + "no_save_rng": null, + "no_weight_decay_cond_type": null, + "non_persistent_ckpt_type": null, + "non_persistent_global_ckpt_dir": null, + "non_persistent_local_ckpt_algo": "fully_parallel", + "non_persistent_local_ckpt_dir": null, + "non_persistent_save_interval": null, + "normalization": "RMSNorm", + "null_tokenizer_eod_id": null, + "null_tokenizer_pad_id": -1, + "num_attention_heads": 11, + "num_channels": 3, + "num_classes": 1000, + "num_dataset_builder_threads": 1, + "num_distributed_optimizer_instances": 1, + "num_experts": null, + "num_floating_point_operations_so_far": 0, + "num_layers": 20, + "num_layers_at_end_in_bf16": 1, + "num_layers_at_start_in_bf16": 1, + "num_layers_per_virtual_pipeline_stage": null, + "num_query_groups": 11, + "num_speculative_tokens": 0, + "num_virtual_stages_per_pipeline_rank": null, + "num_workers": 0, + "nvtx_ranges": false, + "object_storage_cache_path": null, + "offload_modules": "[]", + "offset_sampling_seed_by_dp_rank": true, + "one_logger_async": false, + "one_logger_project": "megatron-lm", + "one_logger_run_name": null, + "online_data": true, + "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "online_dataset": "HuggingFaceFW/fineweb-edu", + "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "online_dataset_subset": "sample-100BT", + "online_latency_log_dir": null, + "online_pack_sequences": false, + "online_prefetch_batches": 16, + "online_raw_manifest": null, + "online_shuffle_buffer_records": 128, + "online_source_mode": "stream", + "online_source_split": "train", + "online_split_seed": 260421106, + "online_text_column": "text", + "online_tokenizer_batch_size": 256, + "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "online_validation_fraction": 0.001, + "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json", + "onnx_safe": null, + "openai_gelu": false, + "optimizer": "muonh", + "optimizer_cpu_offload": false, + "optimizer_cuda_graph": false, + "optimizer_offload_fraction": 1.0, + "otel_enabled": false, + "otel_service_name": null, + "otel_span_groups": null, + "outer_dp_sharding_strategy": "no_shard", + "output_bert_embeddings": false, + "overlap_cpu_optimizer_d2h_h2d": false, + "overlap_dispatch_backward_with_experts_wgrad": false, + "overlap_grad_reduce": false, + "overlap_moe_expert_parallel_comm": false, + "overlap_p2p_comm": false, + "overlap_p2p_comm_warmup_flush": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "override_ckpt_iteration": null, + "override_opt_param_scheduler": false, + "pad_vocab_size": true, + "padded_vocab_size": 32064, + "paper_base_matrix_lr": 0.014, + "paper_embedding_lr": 0.3, + "paper_norm_lr": 0.005, + "paper_other_lr": 0.005, + "paper_reference_batch_tokens": 262144, + "paper_reference_tokens": 983315200, + "paper_reference_width": 640, + "paper_transfer": "Paper2604Transfer(width=1408, total_tokens=7000031232, batch_tokens=262144, reference_width=640, reference_tokens=983315200, reference_batch_tokens=262144)", + "paper_unembedding_lr": 0.004, + "params_dtype": "torch.bfloat16", + "patch_dim": 16, + "per_dataset_sequences_path": null, + "per_split_data_args_path": null, + "perform_initialization": true, + "perform_rl_step": false, + "phase_transition_iterations": null, + "pin_cpu_grads": true, + "pin_cpu_params": true, + "pipeline_model_parallel_comm_backend": null, + "pipeline_model_parallel_layout": null, + "pipeline_model_parallel_size": 1, + "position_embedding_type": "rope", + "pretrained_checkpoint": null, + "profile": false, + "profile_ranks": "[]", + "profile_step_end": 12, + "profile_step_start": 10, + "pytorch_profiler_collect_callstack": false, + "pytorch_profiler_collect_chakra": false, + "pytorch_profiler_collect_shapes": false, + "q_lora_rank": null, + "qk_clip": false, + "qk_clip_alpha": 0.5, + "qk_clip_threshold": 100, + "qk_head_dim": 128, + "qk_l2_norm": false, + "qk_layernorm": false, + "qk_pos_emb_head_dim": 64, + "query_in_block_prob": 0.1, + "quick_geglu": false, + "rampup_batch_size": null, + "rank": 0, + "recipe_optimizer": "{'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}", + "recipe_overrides": "{}", + "recipe_requested_batch_tokens": 262144, + "recipe_schedule": "{'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}", + "recompute_granularity": null, + "recompute_method": null, + "recompute_modules": null, + "recompute_num_layers": null, + "record_memory_history": false, + "record_shapes": false, + "refit_execution_batch_bytes": null, + "refit_method": "gloo", + "relative_attention_max_distance": 128, + "relative_attention_num_buckets": 32, + "replication": false, + "replication_factor": 2, + "replication_jump": null, + "rerun_mode": "validate_results", + "reset_attention_mask": false, + "reset_position_ids": false, + "result_rejected_tracker_filename": null, + "retriever_report_topk_accuracies": "[]", + "retriever_score_scaling": false, + "reuse_grad_buf_for_mxfp8_param_ag": false, + "rl_consumption_granularity": "B", + "rl_default_temperature": 1.0, + "rl_default_top_k": -1, + "rl_default_top_p": 0, + "rl_durable_rollout_bank": false, + "rl_generation_lag": null, + "rl_importance_sampling_truncation_coef": null, + "rl_inference_expert_model_parallel_size": null, + "rl_inference_expert_tensor_model_parallel_size": null, + "rl_inference_logprobs_is_correction": false, + "rl_inference_model_unified_memory_level": 0, + "rl_inference_parsers": "[]", + "rl_inference_pipeline_model_parallel_size": null, + "rl_inference_tensor_model_parallel_size": null, + "rl_kv_cache_management_mode": "persist", + "rl_max_inflight_requests": null, + "rl_offload_inference_model_weights_when_idle": false, + "rl_offload_optimizer_during_inference": false, + "rl_partial_rollouts": false, + "rl_persist_cuda_graphs": false, + "rl_profile": false, + "rl_profile_dir": null, + "rl_prompts_per_eval": 32, + "rl_rollout_bank_dir": null, + "rl_rollout_bank_max_bytes": 0, + "rl_sequence_packing_algo": "fifo", + "rl_sequence_packing_max_sequences_per_bin": 50, + "rl_skip_bos_token": false, + "rl_submission_granularity": "B", + "rl_training_cuda_graphs": false, + "rl_use_sequence_packing": false, + "rl_verify_model_weights_swap": false, + "rope_scaling_factor": 8.0, + "rope_type": null, + "rotary_base": 10000, + "rotary_interleaved": false, + "rotary_percent": 1.0, + "rotary_scaling_factor": 1.0, + "rotary_seq_len_interpolation_factor": null, + "run_workload_inspector_server": false, + "sample_rate": 1.0, + "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s16000_to26703/checkpoints", + "save_activations_interval": null, + "save_dgrads_interval": null, + "save_interval": 4000, + "save_params_interval": null, + "save_retain_interval": null, + "save_tokenizer_assets": true, + "save_tokens_per_expert_interval": null, + "save_wgrads_interval": null, + "seed": 42, + "seq_length": 2048, + "sequence_packing_scheduler": null, + "sequence_parallel": false, + "sft": false, + "sft_mock_dataset_config_json": null, + "sft_tokenizer_prompt_format": "nemotron-h-aligned", + "sgd_momentum": 0.9, + "sharp_enabled_group": null, + "short_seq_prob": 0.1, + "skip_train": false, + "skipped_train_samples": 0, + "softmax_type": "vanilla", + "spec": null, + "special_tokens": null, + "split": null, + "squared_relu": false, + "start_eval_at_iter": null, + "start_weight_decay": 0.0, + "step_batch_size_schedule": null, + "straggler_ctrlr_port": 65535, + "straggler_minmax_count": 1, + "strict_fsdp_dtensor_load": true, + "suggested_communication_unit_size": null, + "swiglu": false, + "swin_backbone_type": "tiny", + "symmetric_ar_type": null, + "te_precision_config_file": null, + "te_rng_tracker": false, + "tensor_model_parallel_size": 1, + "tensor_parallel_num_weight_shards": 1, + "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s16000_to26703/tensorboard", + "tensorboard_log_interval": 1, + "tensorboard_queue_size": 1000, + "test_data_path": null, + "test_mode": false, + "tiktoken_num_special_tokens": 1000, + "tiktoken_pattern": null, + "timing_log_level": 0, + "timing_log_option": "minmax", + "titles_data_path": null, + "tokenizer_hf_no_include_special_tokens": false, + "tokenizer_hf_no_use_fast": false, + "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "tokenizer_sentencepiece_ignore_extra_whitespaces": true, + "tokenizer_sentencepiece_legacy": false, + "tokenizer_type": "HuggingFaceTokenizer", + "torch_fsdp2_reshard_after_forward": true, + "tp_comm_bootstrap_backend": "nccl", + "tp_comm_bulk_dgrad": true, + "tp_comm_bulk_wgrad": true, + "tp_comm_overlap": false, + "tp_comm_overlap_ag": true, + "tp_comm_overlap_cfg": null, + "tp_comm_overlap_rs": true, + "tp_comm_overlap_rs_dgrad": false, + "tp_comm_split_ag": true, + "tp_comm_split_rs": true, + "train_data_path": null, + "train_iters": 26703, + "train_samples": null, + "train_sync_interval": null, + "training_recipe": "paper2604_21106", + "training_recipe_metadata": "{'recipe': 'paper2604_21106', 'width': 1408, 'total_tokens': 7000031232, 'batch_tokens': 262144, 'reference_width': 640, 'reference_tokens': 983315200, 'reference_batch_tokens': 262144, 'base_lrs': {'matrix': 0.014, 'embedding': 0.3, 'unembedding': 0.004, 'norm': 0.005, 'other': 0.005}, 'peak_lrs': {'matrix': 0.007470638533717786, 'embedding': 0.2022599587389726, 'unembedding': 0.0026967994498529684, 'norm': 0.00337099931231621, 'other': 0.00337099931231621}, 'reference': 'paper vanilla 0-20-0 at width 640; configurable token anchor', 'tied_embedding_policy': 'unembedding (one AdamW update)', 'other_parameter_policy': 'norm LR (extension for non-paper scalars/biases)', 'weight_decay': 0.0, 'warmup_steps': 0, 'final_lr_fraction': 0.1, 'optimizer': {'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}, 'schedule': {'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}, 'group_weight_decay': {'matrix': 0.0, 'embedding': 0.0, 'unembedding': 0.0, 'norm': 0.0, 'other': 0.0}, 'recipe_overrides': {}, 'requested_batch_tokens': 262144, 'sequence_length': 2048, 'batch_assumptions': {'global_batch_tokens': 262144, 'sequence_length': 2048, 'round_up': False}, 'lr_transfer': 'mup_hyperp_released_source', 'assumptions': [], 'backend_kernel_assumptions': {}, 'paper_settings_overridden': False, 'model': {'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}}", + "transformer_impl": "local", + "transformer_pipeline_model_parallel_size": 1, + "trust_remote_code": false, + "untie_embeddings_and_output_weights": true, + "use_checkpoint_args": false, + "use_checkpoint_opt_param_scheduler": false, + "use_cpu_initialization": null, + "use_dist_ckpt": false, + "use_dist_ckpt_deprecated": false, + "use_distributed_optimizer": false, + "use_flash_attn": false, + "use_fused_mhc": false, + "use_fused_weighted_squared_relu": false, + "use_gigatoken": false, + "use_gloo_process_groups": true, + "use_grouped_gemm_for_dense_mlp": false, + "use_grouped_gemm_for_shared_expert": false, + "use_kitchen_attention": false, + "use_layer_wise_distributed_optimizer": false, + "use_layer_wise_param_layout": true, + "use_legacy_static_engine": false, + "use_mamba_mem_eff_path": true, + "use_megatron_fsdp": false, + "use_mp_args_from_checkpoint_args": false, + "use_mup": false, + "use_one_sent_docs": false, + "use_persistent_ckpt_worker": false, + "use_precision_aware_optimizer": false, + "use_pytorch_profiler": false, + "use_ring_exchange_p2p": false, + "use_rope_scaling": false, + "use_rotary_position_embeddings": false, + "use_sharp": false, + "use_te_activation_func": false, + "use_tokenizer_model_from_checkpoint_args": true, + "use_torch_fsdp2": false, + "use_torch_optimizer_for_cpu_offload": false, + "use_tp_pp_dp_mapping": false, + "use_transformer_engine_op_fuser": false, + "use_varlen_dataset": false, + "v_head_dim": 128, + "valid_data_path": null, + "validation_set_names": null, + "variable_seq_lengths": false, + "varlen_mock_dataset_config_json": null, + "varlen_sbhd_validation": false, + "verify_integrity": false, + "virtual_pipeline_model_parallel_size": null, + "vision_backbone_type": "vit", + "vision_pretraining": false, + "vision_pretraining_type": "classify", + "vocab_extra_ids": 0, + "vocab_file": null, + "vocab_size": 32001, + "wandb_entity": "LoopTF-4-CSPs", + "wandb_exp_name": "r4_la_hf_warmstart_s16000_to26703", + "wandb_project": "loopTF-workspace", + "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s16000_to26703/wandb", + "weight_decay": 0.0, + "weight_decay_incr_style": "constant", + "wgrad_deferral_limit": 0, + "window_attn_skip_freq": null, + "window_size": null, + "world_size": 16, + "yaml_cfg": null, + "yarn_beta_fast": null, + "yarn_beta_slow": null, + "yarn_correction_range_round_to_int": null, + "yarn_original_max_position_embeddings": null + }, + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3967744, + "lookahead_parameters": 93191296, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659335424, + "total_trainable": 659335424 + }, + "pipeline_model_parallel_size": 1, + "run_name": "r4_weights_only_d0_p0_a2_r0_s16000_to26703", + "source_run_directory": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s16000_to26703", + "tensor_model_parallel_size": 1, + "tokenizer": { + "added_tokens": [ + "<|assistant_end|>" + ], + "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training_budget_tokens": 7000031232, + "training_code_revision": "8219ffa08d51818b9b6b150cbee1b35089dc91af", + "training_tokens": 7000031232, + "warm_start_step": 16000 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..e10dbbab8bd39a8b3003247b7e2b55e5886c6447 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/launch_config.env @@ -0,0 +1,14 @@ +# HF safetensors warm start: copied FBW weights, fresh optimizer/RNG/data stream. +source "$(dirname "${BASH_SOURCE[0]}")/lookahead.env" +RUN_NAME=r4_weights_only_d0_p0_a2_r0 +# Continue without activation checkpointing in either the copied backbone or +# the freshly initialized refiner blocks. +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +# The generated native checkpoint carries a fresh optimizer/RNG whose scheduler +# is advanced to the source step. Online-loader state intentionally starts fresh. +ONLINE_SOURCE_MODE=stream +# Reuse the immutable held-out set already qualified by successful multi-node +# Look-ahead runs; streaming validation can leave ranks waiting at different rates. +ONLINE_VALIDATION_MANIFEST="${PROJECT_ROOT}/data/validation/manual_14b/manifest.json" +WANDB_TAGS=product,resume,R4,transition,lookahead,D0,P0,A2,R0,fbw_norms,weights_only,20l,7b diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..858dd7ed72d52c4566454b5f2755fda0adc29c33 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "sha256": "fd67b820fa30d4ee3bb9785b5c44c0039727df21098f9127064d7ccadc4a771c", + "size": 747 + }, + "config.json": { + "sha256": "45d0f3e5de27a04438cb5632ed9bbe2a28b70d31372c787b68771b3db9a25e9f", + "size": 46454 + }, + "launch_config.env": { + "sha256": "74721ad10af3e65776cd7e5270239a979f4ef83b4ef8583eaee6ec30fc49dfd3", + "size": 878 + }, + "model.safetensors": { + "sha256": "4eb3371ed7507e6a6f7b6e7a382cf441526968d36a275793600e86e7130bfdb7", + "size": 1318836848 + }, + "training_recipe.json": { + "sha256": "9ed332689297ca4fbeaeaf5d5f0be3a9f4503792fb6390032e3e650b712c945a", + "size": 4472 + } + }, + "note": "manifest.json does not hash itself" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bbde7a67d3911ffd048b3199d698dbee3dc1f294 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb3371ed7507e6a6f7b6e7a382cf441526968d36a275793600e86e7130bfdb7 +size 1318836848 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..afe10f9fee24a774d3a95588c119129972cf1e42 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart16000-to26703--7t0tefok/step_26703/training_recipe.json @@ -0,0 +1,153 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659335424, + "feedback_parameters": 3967744, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93191296, + "total_parameters": 659335424 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..399ff46c30aca2acd56c3346453cfef6ce464c81 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "wkv2n5ub", + "previous_wandb_name": "r4_la_hf_warmstart_s20000_to26703", + "training_run_name": "r4_la_hf_warmstart_s20000_to26703", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0, + "F": 0, + "normIn": true, + "normOut": true + }, + "seed": 42, + "hf_old_paths": [ + "r4_weights_only_d0_p0_a2_r0_s20000_to26703" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3deb924aed2487c98dfc619f2ca6e6c949cc439f --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/README.md @@ -0,0 +1,11 @@ +# R4 Look-ahead final weights (20000 warm start) + +This is the final step 26,703 weights-only export for `r4_weights_only_d0_p0_a2_r0_s20000_to26703`, warm-started from the FBW export at step 20000. + +## Format + +`model.safetensors` contains the native Look-ahead Megatron model state only. It excludes optimizer, scheduler, RNG, dataloader, caches, logs, and credentials. It is not a Transformers `AutoModel.from_pretrained` checkpoint. + +To load, use the code revision in `config.json`, construct `lookahead_tf.hf.model.LookaheadTransformer` with `model_config` and `lookahead_config`, remove exactly the `model.` prefix from each tensor key, and call `load_state_dict(..., strict=True, assign=True)`. + +Use `manifest.json` to verify payload hashes. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..741c77c7986b475031bab6f65d705f1f01eca28b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/config.json @@ -0,0 +1,1041 @@ +{ + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_version": 3.0, + "excluded_state": [ + "optimizer", + "opt_param_scheduler", + "rng_state", + "dataloader" + ], + "iteration": 26703, + "loading": { + "format": "native PyTorch state dict saved with safetensors; not a Transformers AutoModel", + "instructions": "Construct LookaheadTransformer(model_config, lookahead_config), remove exactly the model. prefix, then load_state_dict(..., strict=True, assign=True).", + "key_prefix": "model." + }, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 0.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "model_arguments": { + "_is_global_batch_size_explicitly_specified": true, + "account_for_embedding_in_pipeline_split": false, + "account_for_loss_in_pipeline_split": false, + "accumulate_allreduce_grads_in_fp32": true, + "activation_func_clamp_value": null, + "activation_offload_fraction": 1.0, + "adam_beta1": 0.8, + "adam_beta2": 0.95, + "adam_eps": 1e-10, + "adaptive_muon_beta2": 0.95, + "adaptive_muon_eps": 1e-10, + "adaptive_muon_moment2_method": "normuon", + "add_bias_linear": false, + "add_position_embedding": true, + "add_qkv_bias": false, + "adlr_autoresume": false, + "adlr_autoresume_interval": 1000, + "align_grad_reduce": true, + "align_param_gather": false, + "allow_ambiguous_pad_tokens": false, + "app_tag_run_name": null, + "app_tag_run_version": "0.0.0", + "apply_query_key_layer_scaling": false, + "apply_residual_connection_post_layernorm": false, + "apply_rope_fusion": false, + "apply_wd_to_qk_layernorm": false, + "architecture": "lookahead", + "architecture_module": "[]", + "async_ckpt_cpu_priority": 10, + "async_ckpt_io_priority": 3, + "async_ckpt_use_cpu_shm": false, + "async_save": false, + "async_strategy": "mcore", + "attention_backend": "AttnBackend.auto", + "attention_cp_layout": "zigzag", + "attention_dropout": 0.0, + "attention_output_gate": false, + "attention_softmax_in_fp32": false, + "auto_detect_ckpt_format": false, + "backbone_metadata": "{'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}", + "backbone_profile": "paper2604_vanilla", + "barrier_with_L1_time": true, + "batch_invariant_backend": "te_native", + "batch_invariant_collective": "ordered", + "batch_invariant_mode": false, + "bert_binary_head": true, + "bert_embedder_type": "megatron", + "bert_load": null, + "bf16": true, + "bias_dropout_fusion": false, + "bias_gelu_fusion": false, + "bias_swiglu_fusion": true, + "biencoder_projection_dim": 0, + "biencoder_shared_query_context_model": false, + "block_data_path": null, + "cache_mla_latents": false, + "calc_ft_timeouts": false, + "calculate_per_token_loss": false, + "chat_template": null, + "check_for_large_grads": false, + "check_for_nan_in_loss_and_grad": true, + "check_for_spiky_loss": false, + "check_weight_hash_across_dp_replicas_interval": null, + "ckpt_assume_constant_structure": false, + "ckpt_convert_format": null, + "ckpt_convert_save": null, + "ckpt_convert_update_legacy_dist_opt_format": false, + "ckpt_drop_redundant_extra_state": false, + "ckpt_format": "torch", + "ckpt_fully_parallel_load": false, + "ckpt_fully_parallel_load_exchange_algo": "broadcast", + "ckpt_fully_parallel_load_per_rank_objects": false, + "ckpt_fully_parallel_load_process_group": "dp", + "ckpt_fully_parallel_save": true, + "ckpt_fully_parallel_save_deprecated": false, + "ckpt_fully_parallel_save_process_group": "dp", + "ckpt_load_validate_sharding_integrity": true, + "ckpt_pg_tensors_cache_create": false, + "ckpt_pg_tensors_cache_path": null, + "ckpt_step": null, + "classes_fraction": 1.0, + "clip_grad": 0.0, + "clone_scatter_output_in_embedding": true, + "config_logger_dir": "", + "consumed_train_samples": 3417984, + "consumed_valid_samples": 68352, + "context_parallel_size": 1, + "cp_comm_type": "['p2p']", + "cpu_offloading_num_layers": 0, + "cpu_offloading_retain_pinned_cpu_buffers": false, + "create_all_gather_group": false, + "create_attention_mask_in_dataloader": false, + "cross_entropy_fusion_impl": "native", + "cross_entropy_loss_fusion": false, + "csa_compress_ratios": null, + "csa_compress_rotary_base": 40000.0, + "csa_dense_mode": false, + "csa_window_size": 128, + "cuda_graph_impl": "none", + "cuda_graph_modules": "[]", + "cuda_graph_warmup_steps": 3, + "curr_iteration": 26702, + "data_args_path": null, + "data_cache_path": null, + "data_parallel_random_init": false, + "data_parallel_sharding_strategy": "optim_grads_params", + "data_parallel_size": 16, + "data_path": null, + "data_per_class_fraction": 1.0, + "data_sharding": true, + "dataloader_defer_npy_index_mmap": false, + "dataloader_fast_cache_load": false, + "dataloader_inter_document_masking": false, + "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s20000_to26703/dataloader", + "dataloader_type": "external", + "ddp_average_in_collective": false, + "ddp_bucket_size": null, + "ddp_num_buckets": null, + "ddp_pad_buckets_for_high_nccl_busbw": false, + "ddp_param_name_patterns_for_fp32_local_accumulation": "[]", + "ddp_reduce_scatter_with_fp32_accumulation": false, + "decode_only_cuda_graphs": false, + "decoder_first_pipeline_num_layers": null, + "decoder_last_pipeline_num_layers": null, + "decoder_num_layers": null, + "decoder_seq_length": null, + "decoupled_lr": null, + "decoupled_min_lr": null, + "decrease_batch_size_if_needed": false, + "defer_embedding_wgrad_compute": false, + "delay_offload_until_cuda_graph": false, + "delay_wgrad_compute": false, + "delta_offload_bytes_across_pp_ranks": 0, + "deprecated_use_mcore_models": false, + "deterministic_mode": false, + "dino_bottleneck_size": 256, + "dino_freeze_last_layer": 1, + "dino_head_hidden_size": 2048, + "dino_local_crops_number": 10, + "dino_local_img_size": 96, + "dino_norm_last_layer": false, + "dino_teacher_temp": 0.07, + "dino_warmup_teacher_temp": 0.04, + "dino_warmup_teacher_temp_epochs": 30, + "disable_bf16_reduced_precision_matmul": false, + "disable_jit_fuser": false, + "disable_msc_deprecated": false, + "disable_straggler_on_startup": false, + "disable_symmetric_registration": false, + "dist_ckpt_format_deprecated": null, + "dist_ckpt_optim_fully_reshardable": false, + "dist_ckpt_save_pre_mcore_014": false, + "dist_ckpt_strictness": "assume_ok_unexpected", + "dist_ckpt_workers": 1, + "distrib_optim_fully_reshardable_mem_efficient": false, + "distribute_saved_activations": false, + "distributed_backend": "nccl", + "distributed_timeout_minutes": 10, + "distributed_timeout_seconds_after_init": null, + "do_test": 0, + "do_train": 1, + "do_valid": 1, + "dsa_indexer_head_dim": null, + "dsa_indexer_k_norm_epsilon": null, + "dsa_indexer_k_norm_fp32": false, + "dsa_indexer_loss_coeff": null, + "dsa_indexer_n_heads": null, + "dsa_indexer_rope_interleaved": false, + "dsa_indexer_rotate_activation": true, + "dsa_indexer_scoring_relu": true, + "dsa_indexer_skip_topk_offset": 0, + "dsa_indexer_topk": null, + "dsa_indexer_topk_freq": 1, + "dsa_indexer_use_sparse_loss": false, + "dsa_kernel_backend": "none", + "dump_param_to_param_group_map": null, + "embedding_init_method_std": null, + "embedding_path": null, + "empty_unused_memory_level": 0, + "enable_chunked_prefill": false, + "enable_cuda_graph": false, + "enable_experimental": false, + "enable_ft_package": false, + "enable_full_sharding_in_hsdp": false, + "enable_mhc_connections": false, + "enable_msc": false, + "enable_one_logger": true, + "encoder_num_layers": 20, + "encoder_seq_length": 2048, + "end_weight_decay": 0.0, + "eod_mask_loss": false, + "ep_overlap_early_attn_memory_release": false, + "error_injection_rate": 0, + "error_injection_type": "transient_error", + "eval_at_start": true, + "eval_global_batch_size": 128, + "eval_interval": 100, + "eval_iters": 2, + "eval_micro_batch_size": 8, + "evaluation_metadata": "{'eval_at_start': True, 'initial_iters': 2, 'intermediate_iters': 2, 'final_validation_iters': 270, 'test_iters': 0, 'global_batch_size': 128, 'sequence_length': 2048, 'intermediate_input_tokens': 524288, 'final_validation_sequences': 34560, 'final_validation_input_tokens': 70778880, 'validation_iterator': 'continues_existing_held_out_iterator'}", + "evidence_data_path": null, + "exit_duration_in_mins": null, + "exit_interval": 26703, + "exit_on_missing_checkpoint": true, + "exit_signal": 15, + "exit_signal_handler": false, + "exit_signal_handler_for_dataloader": false, + "exit_signal_handler_for_training": false, + "exp_avg_dtype": "torch.float32", + "exp_avg_sq_dtype": "torch.float32", + "experimental_attention_variant": null, + "expert_data_parallel_sharding_strategy": null, + "expert_gtp_weight_remat_size": 1, + "expert_model_parallel_size": 1, + "expert_tensor_parallel_num_weight_shards": 1, + "expert_tensor_parallel_size": 1, + "external_cuda_graph": false, + "fake_process_group": false, + "fault_injector_delay_start_iteration": null, + "fault_injector_fault_delay": null, + "fault_injector_fault_probabilities": null, + "fault_injector_fault_types": null, + "fault_injector_mtti_seconds": null, + "fault_injector_num_ranks": null, + "fault_injector_offset_seconds": null, + "fault_injector_ranks": null, + "fault_injector_seed": null, + "feedback_normuon_beta2": 0.95, + "feedback_scalar_lr": 0.0005, + "feedback_scalar_weight_decay": 0.0, + "feedback_z_loss_coefficient": 0.0, + "feedback_z_loss_start_fraction": 1.0, + "ffn_hidden_size": 5632, + "fim_data": false, + "fim_eod_token": "<|endoftext|>", + "fim_fragment_rate": null, + "fim_middle_token": "", + "fim_no_prefix": null, + "fim_pad_token": "", + "fim_prefix_token": "", + "fim_rate": 0.5, + "fim_split_sample": null, + "fim_spm_rate": 0.5, + "fim_suffix_token": "", + "final_eval_iters": 270, + "fine_grained_activation_offloading": false, + "fine_grained_offloading_max_inflight_offloads": null, + "finetune": false, + "first_last_layers_bf16": false, + "flash_attention_version": null, + "flash_decode": false, + "flight_recorder_dump_on_timeout": true, + "flight_recorder_dump_path": null, + "flight_recorder_extra_dump_on_exec": true, + "flight_recorder_include_only_active": true, + "flight_recorder_include_stack_trace": false, + "flight_recorder_trace_buffer_size": 2000, + "fp16": false, + "fp16_lm_cross_entropy": false, + "fp32_residual_connection": false, + "fp4": null, + "fp4_param_gather": false, + "fp4_quantizer_factory": null, + "fp4_recipe": "nvfp4", + "fp8": null, + "fp8_amax_compute_algo": "most_recent", + "fp8_amax_history_len": 1, + "fp8_interval": 1, + "fp8_margin": 0, + "fp8_output_proj": false, + "fp8_param_gather": false, + "fp8_quantizer_factory": null, + "fp8_recipe": "delayed", + "fp8_wgrad": true, + "freeze_all_layers": false, + "fsdp_db_use_persist_buf_on_alloc_fail": false, + "fsdp_double_buffer": false, + "fsdp_manual_registration": false, + "ft_num_warmup_iters": 5, + "full_validation": false, + "fused_residual_rmsnorm": false, + "gdp_cutedsl_kernel": false, + "gdp_num_chunk_states_to_recompute": 2, + "gdp_num_householder": 3, + "global_batch_size": 128, + "glu_linear_offset": 0.0, + "gpu_sniff_test_interval": null, + "grad_reduce_in_bf16": false, + "gradient_accumulation_fusion": false, + "gradient_reduce_div_fusion": true, + "group_query_attention": false, + "grpo_clamp_eps_lower": 0.01, + "grpo_clamp_eps_upper": 0.01, + "grpo_entropy_term_weight": 0.0, + "grpo_filter_groups_with_same_reward": false, + "grpo_group_size": 2, + "grpo_iterations": 2, + "grpo_kl_beta": 0.001, + "grpo_prompts_per_step": 32, + "gtp_expert_remat_nccl_ub": false, + "gtp_remat_nccl_ub": false, + "gtp_remat_opt_in_modules": "[]", + "gtp_remat_reduce_scatter_with_fp32_accumulation": false, + "gtp_weight_remat_size": 1, + "head_lr_mult": 1.0, + "heterogeneous_layers_config_encoded_json": null, + "heterogeneous_layers_config_path": null, + "hfsdp_param_gather_overlap": false, + "hidden_dropout": 0.0, + "hidden_size": 1408, + "hierarchical_context_parallel_sizes": null, + "high_priority_a2a_comm_stream": false, + "high_priority_stream_groups": "[]", + "hybrid_context_parallel": false, + "hybrid_layer_pattern": null, + "hybrid_override_pattern": null, + "hysteresis": 2, + "ict_head_size": null, + "ict_load": null, + "img_h": 224, + "img_w": 224, + "indexer_batch_size": 128, + "indexer_log_interval": 1000, + "inference_batch_times_seqlen_threshold": -1, + "inference_coordinator_port": null, + "inference_cuda_graph_all_prefills": false, + "inference_cuda_graph_max_tokens": 512, + "inference_cuda_graph_scope": "InferenceCudaGraphScope.none", + "inference_disable_ep_consensus": false, + "inference_disable_triton_nvls_kernels": false, + "inference_dynamic_batching": false, + "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, + "inference_dynamic_batching_async_sched_mode": "async", + "inference_dynamic_batching_block_size": 256, + "inference_dynamic_batching_buffer_size_gb": 40.0, + "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, + "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", + "inference_dynamic_batching_enable_prefix_caching": false, + "inference_dynamic_batching_logprobs_mode": "raw_logprobs", + "inference_dynamic_batching_mamba_memory_ratio": null, + "inference_dynamic_batching_max_requests": null, + "inference_dynamic_batching_max_tokens": null, + "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", + "inference_dynamic_batching_media_cache_routing_weight": 1.0, + "inference_dynamic_batching_num_cuda_graphs": 16, + "inference_dynamic_batching_paused_buffer_size_gb": null, + "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, + "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", + "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", + "inference_dynamic_batching_prefix_caching_mamba_gb": null, + "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, + "inference_dynamic_batching_sampling_backend": "torch", + "inference_dynamic_batching_track_generated_token_events": false, + "inference_dynamic_batching_track_paused_request_events": false, + "inference_dynamic_batching_unified_memory_level": 0, + "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, + "inference_flashinfer_mxfp8_token_capacity": null, + "inference_fuse_tp_communication": false, + "inference_grouped_gemm_backend": "vllm", + "inference_logging_step_interval": 0, + "inference_max_requests": 8, + "inference_max_seq_length": 2560, + "inference_moe_disable_fused_quant_kernels": false, + "inference_moe_token_dispatcher_type": "nvls", + "inference_rng_tracker": false, + "inference_shards": null, + "inference_text_gen_server_logging": false, + "inference_use_synchronous_zmq_collectives": false, + "inference_wandb_logging": false, + "init_method_std": 0.02, + "init_method_xavier_uniform": false, + "init_model_with_meta_device": false, + "initial_loss_scale": 4294967296, + "inprocess_active_world_size": 16, + "inprocess_barrier_timeout": 120, + "inprocess_completion_timeout": 120, + "inprocess_empty_cuda_cache": false, + "inprocess_granularity": "node", + "inprocess_hard_timeout": 90, + "inprocess_heartbeat_interval": 30, + "inprocess_heartbeat_timeout": 60, + "inprocess_last_call_wait": 1, + "inprocess_max_iterations": null, + "inprocess_monitor_process_interval": 1.0, + "inprocess_monitor_thread_interval": 1.0, + "inprocess_progress_watchdog_interval": 1.0, + "inprocess_restart": false, + "inprocess_soft_timeout": 60, + "inprocess_termination_grace_time": 1, + "is_hybrid_model": false, + "iter_per_epoch": 1250, + "iteration": 20000, + "iterations_to_skip": "[]", + "keep_fp8_transpose_cache": false, + "kitchen_attention_backend": "sdpa", + "kv_channels": 128, + "kv_lora_rank": 32, + "langrl_env_config": null, + "layernorm_epsilon": 1e-06, + "layernorm_zero_centered_gamma": false, + "lazy_mpu_init": false, + "linear_attention_freq": null, + "linear_conv_kernel_dim": 4, + "linear_cp_layout": "zigzag", + "linear_cp_mode": "headwise", + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "lion_beta1": 0.95, + "lion_beta2": 0.98, + "load": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/transfers/hf_weights_only/s20000/lookahead/checkpoints", + "load_main_params_from_ckpt": false, + "local_rank": 0, + "log_device_memory_used": false, + "log_energy": false, + "log_interval": 1, + "log_loss_scale_to_tensorboard": true, + "log_max_attention_logit": false, + "log_memory_interval": null, + "log_memory_to_tensorboard": false, + "log_num_zeros_in_grad": false, + "log_params_norm": false, + "log_progress": false, + "log_straggler": false, + "log_throughput": false, + "log_timers_to_tensorboard": true, + "log_validation_ppl_to_tensorboard": false, + "log_world_size_to_tensorboard": false, + "logging_level": null, + "logit_dtype": null, + "logits_load_decode_threads": 4, + "logits_load_dir": null, + "logits_load_ignore_errors": false, + "logits_load_kd_loss_alpha": 1.0, + "logits_load_msc_prefetch_depth": 2, + "logits_load_prefetch_factor": 3, + "logits_save_dir": null, + "logits_save_dtype": "fp16", + "logits_save_top_k": null, + "logits_save_top_p": null, + "logits_save_top_p_min_k": 1, + "lookahead_allow_objective_change_on_resume": false, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_cache_layers": null, + "lookahead_checkpoint_refiner": false, + "lookahead_draft_loss_weight": 0.0, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_full_attention_interval": 6, + "lookahead_future_gradient_mode": "all", + "lookahead_future_loss_weight": 0.0, + "lookahead_gradient_checkpointing": false, + "lookahead_mode": "parallel", + "lookahead_normalize_feedback_embedding": true, + "lookahead_normalize_feedback_output": true, + "lookahead_num_passes": 3, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_temperature": 1.0, + "lookahead_pseudo_top_k": 128, + "lookahead_refined_future_loss_weight": 0.0, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_z_loss_coefficient": 0.0, + "loss_scale": null, + "loss_scale_window": 1000, + "lr": 0.007470638533717786, + "lr_decay_iters": null, + "lr_decay_samples": null, + "lr_decay_style": "linear", + "lr_warmup_fraction": null, + "lr_warmup_init": 0.0, + "lr_warmup_iters": 0, + "lr_warmup_samples": 0, + "lr_wsd_decay_iters": 0, + "lr_wsd_decay_samples": null, + "lr_wsd_decay_style": "linear", + "main_grads_dtype": "torch.float32", + "main_params_dtype": "torch.float32", + "make_vocab_size_divisible_by": 64, + "mamba_head_dim": 64, + "mamba_inference_conv_states_dtype": "torch.bfloat16", + "mamba_inference_ssm_states_dtype": "torch.bfloat16", + "mamba_num_groups": 8, + "mamba_num_heads": null, + "mamba_state_dim": 128, + "mamba_training_ssm_states_dtype": null, + "manual_gc": false, + "manual_gc_eval": true, + "manual_gc_interval": 0, + "mask_factor": 1.0, + "mask_prob": 0.15, + "mask_type": "random", + "masked_softmax_fusion": true, + "max_position_embeddings": 2048, + "max_seqlen_per_dp_cp_rank": null, + "max_tokens_to_oom": 12000, + "megatron_fsdp_enable_fine_grained_param_gather": false, + "megatron_fsdp_grad_comm_dtype": null, + "megatron_fsdp_main_grads_dtype": null, + "megatron_fsdp_main_params_dtype": "torch.float32", + "megatron_fsdp_max_pool_double_buffer": false, + "megatron_fsdp_version": 1, + "megatron_lm_path": "/root/dl/runtime/Megatron-LM", + "memory_snapshot_path": "snapshot.pickle", + "merge_file": null, + "metadata_path": null, + "mhc_fused_backend": "auto", + "mhc_init_gating_factor": 0.01, + "mhc_num_residual_streams": 4, + "mhc_recompute_layer_num": null, + "mhc_sinkhorn_iterations": 20, + "micro_batch_size": 8, + "microbatch_group_size_per_vp_stage": null, + "mid_level_dataset_surplus": 0.005, + "min_loss_scale": 1.0, + "min_lr": 0.0007470638533717787, + "min_offloaded_tensor_size": 1048576, + "mla_down_proj_fusion": false, + "mlp_chunks_for_prefill": 1, + "mlp_chunks_for_training": 1, + "mmap_bin_files": true, + "mock_data": false, + "moe_apply_probs_on_input": false, + "moe_aux_loss_coeff": 0.0, + "moe_combine_bwd_dtype": "bf16", + "moe_deepep_num_sms": null, + "moe_dispatch_fwd_dtype": "bf16", + "moe_enable_deepep": false, + "moe_enable_routing_replay": false, + "moe_expert_capacity_factor": null, + "moe_expert_rank_capacity_factor": null, + "moe_ffn_hidden_size": null, + "moe_flex_dispatcher_backend": "deepep", + "moe_flex_dispatcher_num_sms": null, + "moe_grouped_gemm": false, + "moe_hybridep_num_blocks_permute": null, + "moe_hybridep_num_blocks_unpermute": null, + "moe_hybridep_num_sms": null, + "moe_hybridep_num_sms_preprocessing": 108, + "moe_hybridep_pad_uneven_dispatch_inputs": false, + "moe_input_jitter_eps": null, + "moe_latent_size": null, + "moe_layer_freq": 1, + "moe_layer_recompute": false, + "moe_mlp_glu_interleave_size": null, + "moe_ncclep_zero_copy": false, + "moe_pad_expert_input_to_capacity": false, + "moe_pad_experts_for_cuda_graph_inference": false, + "moe_paged_stash": false, + "moe_paged_stash_buffer_size_factor_cpu": 0.0, + "moe_paged_stash_buffer_size_factor_cuda": 1.1, + "moe_paged_stash_page_size": 64, + "moe_per_layer_logging": false, + "moe_permute_fusion": false, + "moe_permute_fusion_into_hybridep": false, + "moe_router_bias_update_rate": 0.001, + "moe_router_dtype": null, + "moe_router_enable_expert_bias": false, + "moe_router_force_biased": null, + "moe_router_force_load_balancing": false, + "moe_router_fusion": false, + "moe_router_group_topk": null, + "moe_router_load_balancing_type": "aux_loss", + "moe_router_num_groups": null, + "moe_router_padding_for_fp8": false, + "moe_router_padding_for_quantization": false, + "moe_router_pre_softmax": false, + "moe_router_quantile_balancing_ema": 0.0, + "moe_router_score_function": "softmax", + "moe_router_skip_muon": false, + "moe_router_topk": 2, + "moe_router_topk_scaling_factor": null, + "moe_routing_trace_capture_hidden_states": false, + "moe_routing_trace_capture_logits": false, + "moe_routing_trace_dump_weights": false, + "moe_routing_trace_max_training_iters": null, + "moe_routing_trace_path": null, + "moe_shared_expert_gate": false, + "moe_shared_expert_glu_interleave_size": null, + "moe_shared_expert_intermediate_size": null, + "moe_shared_expert_overlap": false, + "moe_single_grouped_bias": false, + "moe_single_grouped_weight": false, + "moe_token_dispatcher_type": "allgather", + "moe_token_drop_policy": "probs", + "moe_upcycling_granularity": 1, + "moe_use_grouped_tensor": false, + "moe_use_norm_before_up_proj": false, + "moe_use_upcycling": false, + "moe_z_loss_coeff": null, + "mrope_section": null, + "mscale": 1.0, + "mscale_all_dim": 0.0, + "mtp_detach_heads": false, + "mtp_hsm": false, + "mtp_hybrid_override_pattern": null, + "mtp_loss_scaling_factor": 0.1, + "mtp_num_layers": null, + "mtp_standalone": false, + "mtp_use_repeated_layer": false, + "multi_latent_attention": false, + "multiple_validation_sets": false, + "muon_coefficient_type": "quintic", + "muon_extra_scale_factor": 1.0, + "muon_fp32_matmul_prec": "medium", + "muon_momentum": 0.95, + "muon_nesterov": true, + "muon_num_ns_steps": 5, + "muon_scalar_optimizer": "adam", + "muon_scale_mode": "spectral", + "muon_split_qkv": true, + "muon_tp_mode": "duplicated", + "muon_use_syrk": false, + "mup_attn_scale_power": 1.0, + "mup_base_head_dim": null, + "mup_base_hidden_size": null, + "mup_embedding_mult": 1.0, + "mup_output_mult": 1.0, + "mup_width_mult": 1.0, + "nccl_all_reduce_for_prefill": false, + "nccl_communicator_config_path": null, + "nccl_ub": false, + "no_load_optim": null, + "no_load_rng": null, + "no_persist_layer_norm": false, + "no_rope_freq": null, + "no_save_optim": null, + "no_save_rng": null, + "no_weight_decay_cond_type": null, + "non_persistent_ckpt_type": null, + "non_persistent_global_ckpt_dir": null, + "non_persistent_local_ckpt_algo": "fully_parallel", + "non_persistent_local_ckpt_dir": null, + "non_persistent_save_interval": null, + "normalization": "RMSNorm", + "null_tokenizer_eod_id": null, + "null_tokenizer_pad_id": -1, + "num_attention_heads": 11, + "num_channels": 3, + "num_classes": 1000, + "num_dataset_builder_threads": 1, + "num_distributed_optimizer_instances": 1, + "num_experts": null, + "num_floating_point_operations_so_far": 0, + "num_layers": 20, + "num_layers_at_end_in_bf16": 1, + "num_layers_at_start_in_bf16": 1, + "num_layers_per_virtual_pipeline_stage": null, + "num_query_groups": 11, + "num_speculative_tokens": 0, + "num_virtual_stages_per_pipeline_rank": null, + "num_workers": 0, + "nvtx_ranges": false, + "object_storage_cache_path": null, + "offload_modules": "[]", + "offset_sampling_seed_by_dp_rank": true, + "one_logger_async": false, + "one_logger_project": "megatron-lm", + "one_logger_run_name": null, + "online_data": true, + "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", + "online_dataset": "HuggingFaceFW/fineweb-edu", + "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "online_dataset_subset": "sample-100BT", + "online_latency_log_dir": null, + "online_pack_sequences": false, + "online_prefetch_batches": 16, + "online_raw_manifest": null, + "online_shuffle_buffer_records": 128, + "online_source_mode": "stream", + "online_source_split": "train", + "online_split_seed": 260421106, + "online_text_column": "text", + "online_tokenizer_batch_size": 256, + "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "online_validation_fraction": 0.001, + "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json", + "onnx_safe": null, + "openai_gelu": false, + "optimizer": "muonh", + "optimizer_cpu_offload": false, + "optimizer_cuda_graph": false, + "optimizer_offload_fraction": 1.0, + "otel_enabled": false, + "otel_service_name": null, + "otel_span_groups": null, + "outer_dp_sharding_strategy": "no_shard", + "output_bert_embeddings": false, + "overlap_cpu_optimizer_d2h_h2d": false, + "overlap_dispatch_backward_with_experts_wgrad": false, + "overlap_grad_reduce": false, + "overlap_moe_expert_parallel_comm": false, + "overlap_p2p_comm": false, + "overlap_p2p_comm_warmup_flush": false, + "overlap_param_gather": false, + "overlap_param_gather_with_optimizer_step": false, + "override_ckpt_iteration": null, + "override_opt_param_scheduler": false, + "pad_vocab_size": true, + "padded_vocab_size": 32064, + "paper_base_matrix_lr": 0.014, + "paper_embedding_lr": 0.3, + "paper_norm_lr": 0.005, + "paper_other_lr": 0.005, + "paper_reference_batch_tokens": 262144, + "paper_reference_tokens": 983315200, + "paper_reference_width": 640, + "paper_transfer": "Paper2604Transfer(width=1408, total_tokens=7000031232, batch_tokens=262144, reference_width=640, reference_tokens=983315200, reference_batch_tokens=262144)", + "paper_unembedding_lr": 0.004, + "params_dtype": "torch.bfloat16", + "patch_dim": 16, + "per_dataset_sequences_path": null, + "per_split_data_args_path": null, + "perform_initialization": true, + "perform_rl_step": false, + "phase_transition_iterations": null, + "pin_cpu_grads": true, + "pin_cpu_params": true, + "pipeline_model_parallel_comm_backend": null, + "pipeline_model_parallel_layout": null, + "pipeline_model_parallel_size": 1, + "position_embedding_type": "rope", + "pretrained_checkpoint": null, + "profile": false, + "profile_ranks": "[]", + "profile_step_end": 12, + "profile_step_start": 10, + "pytorch_profiler_collect_callstack": false, + "pytorch_profiler_collect_chakra": false, + "pytorch_profiler_collect_shapes": false, + "q_lora_rank": null, + "qk_clip": false, + "qk_clip_alpha": 0.5, + "qk_clip_threshold": 100, + "qk_head_dim": 128, + "qk_l2_norm": false, + "qk_layernorm": false, + "qk_pos_emb_head_dim": 64, + "query_in_block_prob": 0.1, + "quick_geglu": false, + "rampup_batch_size": null, + "rank": 0, + "recipe_optimizer": "{'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}", + "recipe_overrides": "{}", + "recipe_requested_batch_tokens": 262144, + "recipe_schedule": "{'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}", + "recompute_granularity": null, + "recompute_method": null, + "recompute_modules": null, + "recompute_num_layers": null, + "record_memory_history": false, + "record_shapes": false, + "refit_execution_batch_bytes": null, + "refit_method": "gloo", + "relative_attention_max_distance": 128, + "relative_attention_num_buckets": 32, + "replication": false, + "replication_factor": 2, + "replication_jump": null, + "rerun_mode": "validate_results", + "reset_attention_mask": false, + "reset_position_ids": false, + "result_rejected_tracker_filename": null, + "retriever_report_topk_accuracies": "[]", + "retriever_score_scaling": false, + "reuse_grad_buf_for_mxfp8_param_ag": false, + "rl_consumption_granularity": "B", + "rl_default_temperature": 1.0, + "rl_default_top_k": -1, + "rl_default_top_p": 0, + "rl_durable_rollout_bank": false, + "rl_generation_lag": null, + "rl_importance_sampling_truncation_coef": null, + "rl_inference_expert_model_parallel_size": null, + "rl_inference_expert_tensor_model_parallel_size": null, + "rl_inference_logprobs_is_correction": false, + "rl_inference_model_unified_memory_level": 0, + "rl_inference_parsers": "[]", + "rl_inference_pipeline_model_parallel_size": null, + "rl_inference_tensor_model_parallel_size": null, + "rl_kv_cache_management_mode": "persist", + "rl_max_inflight_requests": null, + "rl_offload_inference_model_weights_when_idle": false, + "rl_offload_optimizer_during_inference": false, + "rl_partial_rollouts": false, + "rl_persist_cuda_graphs": false, + "rl_profile": false, + "rl_profile_dir": null, + "rl_prompts_per_eval": 32, + "rl_rollout_bank_dir": null, + "rl_rollout_bank_max_bytes": 0, + "rl_sequence_packing_algo": "fifo", + "rl_sequence_packing_max_sequences_per_bin": 50, + "rl_skip_bos_token": false, + "rl_submission_granularity": "B", + "rl_training_cuda_graphs": false, + "rl_use_sequence_packing": false, + "rl_verify_model_weights_swap": false, + "rope_scaling_factor": 8.0, + "rope_type": null, + "rotary_base": 10000, + "rotary_interleaved": false, + "rotary_percent": 1.0, + "rotary_scaling_factor": 1.0, + "rotary_seq_len_interpolation_factor": null, + "run_workload_inspector_server": false, + "sample_rate": 1.0, + "save": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s20000_to26703/checkpoints", + "save_activations_interval": null, + "save_dgrads_interval": null, + "save_interval": 4000, + "save_params_interval": null, + "save_retain_interval": null, + "save_tokenizer_assets": true, + "save_tokens_per_expert_interval": null, + "save_wgrads_interval": null, + "seed": 42, + "seq_length": 2048, + "sequence_packing_scheduler": null, + "sequence_parallel": false, + "sft": false, + "sft_mock_dataset_config_json": null, + "sft_tokenizer_prompt_format": "nemotron-h-aligned", + "sgd_momentum": 0.9, + "sharp_enabled_group": null, + "short_seq_prob": 0.1, + "skip_train": false, + "skipped_train_samples": 0, + "softmax_type": "vanilla", + "spec": null, + "special_tokens": null, + "split": null, + "squared_relu": false, + "start_eval_at_iter": null, + "start_weight_decay": 0.0, + "step_batch_size_schedule": null, + "straggler_ctrlr_port": 65535, + "straggler_minmax_count": 1, + "strict_fsdp_dtensor_load": true, + "suggested_communication_unit_size": null, + "swiglu": false, + "swin_backbone_type": "tiny", + "symmetric_ar_type": null, + "te_precision_config_file": null, + "te_rng_tracker": false, + "tensor_model_parallel_size": 1, + "tensor_parallel_num_weight_shards": 1, + "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s20000_to26703/tensorboard", + "tensorboard_log_interval": 1, + "tensorboard_queue_size": 1000, + "test_data_path": null, + "test_mode": false, + "tiktoken_num_special_tokens": 1000, + "tiktoken_pattern": null, + "timing_log_level": 0, + "timing_log_option": "minmax", + "titles_data_path": null, + "tokenizer_hf_no_include_special_tokens": false, + "tokenizer_hf_no_use_fast": false, + "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "tokenizer_sentencepiece_ignore_extra_whitespaces": true, + "tokenizer_sentencepiece_legacy": false, + "tokenizer_type": "HuggingFaceTokenizer", + "torch_fsdp2_reshard_after_forward": true, + "tp_comm_bootstrap_backend": "nccl", + "tp_comm_bulk_dgrad": true, + "tp_comm_bulk_wgrad": true, + "tp_comm_overlap": false, + "tp_comm_overlap_ag": true, + "tp_comm_overlap_cfg": null, + "tp_comm_overlap_rs": true, + "tp_comm_overlap_rs_dgrad": false, + "tp_comm_split_ag": true, + "tp_comm_split_rs": true, + "train_data_path": null, + "train_iters": 26703, + "train_samples": null, + "train_sync_interval": null, + "training_recipe": "paper2604_21106", + "training_recipe_metadata": "{'recipe': 'paper2604_21106', 'width': 1408, 'total_tokens': 7000031232, 'batch_tokens': 262144, 'reference_width': 640, 'reference_tokens': 983315200, 'reference_batch_tokens': 262144, 'base_lrs': {'matrix': 0.014, 'embedding': 0.3, 'unembedding': 0.004, 'norm': 0.005, 'other': 0.005}, 'peak_lrs': {'matrix': 0.007470638533717786, 'embedding': 0.2022599587389726, 'unembedding': 0.0026967994498529684, 'norm': 0.00337099931231621, 'other': 0.00337099931231621}, 'reference': 'paper vanilla 0-20-0 at width 640; configurable token anchor', 'tied_embedding_policy': 'unembedding (one AdamW update)', 'other_parameter_policy': 'norm LR (extension for non-paper scalars/biases)', 'weight_decay': 0.0, 'warmup_steps': 0, 'final_lr_fraction': 0.1, 'optimizer': {'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}, 'schedule': {'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}, 'group_weight_decay': {'matrix': 0.0, 'embedding': 0.0, 'unembedding': 0.0, 'norm': 0.0, 'other': 0.0}, 'recipe_overrides': {}, 'requested_batch_tokens': 262144, 'sequence_length': 2048, 'batch_assumptions': {'global_batch_tokens': 262144, 'sequence_length': 2048, 'round_up': False}, 'lr_transfer': 'mup_hyperp_released_source', 'assumptions': [], 'backend_kernel_assumptions': {}, 'paper_settings_overridden': False, 'model': {'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 659335424, 'feedback_parameters': 3967744, 'backbone_total_trainable': 566144128, 'lookahead_parameters': 93191296, 'total_parameters': 659335424}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'lookahead', 'objective': 'lookahead_parallel_next_token', 'lookahead_mode': 'parallel', 'train_passes': 3, 'feedback_loss_weight': 1.0, 'cache_layer_indices': [0, 10, 19], 'refiner_activation': 'silu_gated', 'pseudo_backend': 'sdpa_auto', 'activation_checkpointing': False, 'checkpoint_refiner': False, 'block_application_multiplier': 6.449999999999999, 'future_loss_weight': 0.0, 'future_gradient_mode': 'all', 'draft_loss_weight': 0.0, 'refined_future_loss_weight': 0.0, 'pseudo_initializer': 'A2', 'pseudo_top_k': 128, 'pseudo_temperature': 1.0, 'normalize_feedback_embedding': True, 'normalize_feedback_output': True, 'refiner_variant': 'R0', 'allow_objective_change_on_resume': False}}", + "transformer_impl": "local", + "transformer_pipeline_model_parallel_size": 1, + "trust_remote_code": false, + "untie_embeddings_and_output_weights": true, + "use_checkpoint_args": false, + "use_checkpoint_opt_param_scheduler": false, + "use_cpu_initialization": null, + "use_dist_ckpt": false, + "use_dist_ckpt_deprecated": false, + "use_distributed_optimizer": false, + "use_flash_attn": false, + "use_fused_mhc": false, + "use_fused_weighted_squared_relu": false, + "use_gigatoken": false, + "use_gloo_process_groups": true, + "use_grouped_gemm_for_dense_mlp": false, + "use_grouped_gemm_for_shared_expert": false, + "use_kitchen_attention": false, + "use_layer_wise_distributed_optimizer": false, + "use_layer_wise_param_layout": true, + "use_legacy_static_engine": false, + "use_mamba_mem_eff_path": true, + "use_megatron_fsdp": false, + "use_mp_args_from_checkpoint_args": false, + "use_mup": false, + "use_one_sent_docs": false, + "use_persistent_ckpt_worker": false, + "use_precision_aware_optimizer": false, + "use_pytorch_profiler": false, + "use_ring_exchange_p2p": false, + "use_rope_scaling": false, + "use_rotary_position_embeddings": false, + "use_sharp": false, + "use_te_activation_func": false, + "use_tokenizer_model_from_checkpoint_args": true, + "use_torch_fsdp2": false, + "use_torch_optimizer_for_cpu_offload": false, + "use_tp_pp_dp_mapping": false, + "use_transformer_engine_op_fuser": false, + "use_varlen_dataset": false, + "v_head_dim": 128, + "valid_data_path": null, + "validation_set_names": null, + "variable_seq_lengths": false, + "varlen_mock_dataset_config_json": null, + "varlen_sbhd_validation": false, + "verify_integrity": false, + "virtual_pipeline_model_parallel_size": null, + "vision_backbone_type": "vit", + "vision_pretraining": false, + "vision_pretraining_type": "classify", + "vocab_extra_ids": 0, + "vocab_file": null, + "vocab_size": 32001, + "wandb_entity": "LoopTF-4-CSPs", + "wandb_exp_name": "r4_la_hf_warmstart_s20000_to26703", + "wandb_project": "loopTF-workspace", + "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s20000_to26703/wandb", + "weight_decay": 0.0, + "weight_decay_incr_style": "constant", + "wgrad_deferral_limit": 0, + "window_attn_skip_freq": null, + "window_size": null, + "world_size": 16, + "yaml_cfg": null, + "yarn_beta_fast": null, + "yarn_beta_slow": null, + "yarn_correction_range_round_to_int": null, + "yarn_original_max_position_embeddings": null + }, + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3967744, + "lookahead_parameters": 93191296, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659335424, + "total_trainable": 659335424 + }, + "pipeline_model_parallel_size": 1, + "run_name": "r4_weights_only_d0_p0_a2_r0_s20000_to26703", + "source_run_directory": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/output/research/R4_FBW_Architecture_Transition/runs/r4_la_hf_warmstart_s20000_to26703", + "tensor_model_parallel_size": 1, + "tokenizer": { + "added_tokens": [ + "<|assistant_end|>" + ], + "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "training_budget_tokens": 7000031232, + "training_code_revision": "8219ffa08d51818b9b6b150cbee1b35089dc91af", + "training_tokens": 7000031232, + "warm_start_step": 20000 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..e10dbbab8bd39a8b3003247b7e2b55e5886c6447 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/launch_config.env @@ -0,0 +1,14 @@ +# HF safetensors warm start: copied FBW weights, fresh optimizer/RNG/data stream. +source "$(dirname "${BASH_SOURCE[0]}")/lookahead.env" +RUN_NAME=r4_weights_only_d0_p0_a2_r0 +# Continue without activation checkpointing in either the copied backbone or +# the freshly initialized refiner blocks. +ENABLE_RECOMPUTE=false +LOOKAHEAD_CHECKPOINT_REFINER=false +# The generated native checkpoint carries a fresh optimizer/RNG whose scheduler +# is advanced to the source step. Online-loader state intentionally starts fresh. +ONLINE_SOURCE_MODE=stream +# Reuse the immutable held-out set already qualified by successful multi-node +# Look-ahead runs; streaming validation can leave ranks waiting at different rates. +ONLINE_VALIDATION_MANIFEST="${PROJECT_ROOT}/data/validation/manual_14b/manifest.json" +WANDB_TAGS=product,resume,R4,transition,lookahead,D0,P0,A2,R0,fbw_norms,weights_only,20l,7b diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4dcfc2bdd0152324082b38f37a1ee0a30fa8104c --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "sha256": "a9d356822a2188959abcd2f40fedf0e35f45b0338caac40eb5d9ed4593402981", + "size": 747 + }, + "config.json": { + "sha256": "8c2297f279bd2f68627bfd0889ec87b9a58954e3cce17094eb699fc9761d2d44", + "size": 46454 + }, + "launch_config.env": { + "sha256": "74721ad10af3e65776cd7e5270239a979f4ef83b4ef8583eaee6ec30fc49dfd3", + "size": 878 + }, + "model.safetensors": { + "sha256": "12977292628e257141ec2ddb9334e0be182a120150634aab14a194e7af7b0d08", + "size": 1318836848 + }, + "training_recipe.json": { + "sha256": "9ed332689297ca4fbeaeaf5d5f0be3a9f4503792fb6390032e3e650b712c945a", + "size": 4472 + } + }, + "note": "manifest.json does not hash itself" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76b816edc8ddc506a19e09a7ff9d459943fec078 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12977292628e257141ec2ddb9334e0be182a120150634aab14a194e7af7b0d08 +size 1318836848 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..afe10f9fee24a774d3a95588c119129972cf1e42 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-normIn-normOut-warmstart20000-to26703--wkv2n5ub/step_26703/training_recipe.json @@ -0,0 +1,153 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659335424, + "feedback_parameters": 3967744, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93191296, + "total_parameters": 659335424 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "normalize_feedback_embedding": true, + "normalize_feedback_output": true, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..531f97ae38a829a5ef01e1aa3241baaca8e8b035 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "gcwdjm01", + "previous_wandb_name": "r3-a2-r0-d0-p0p2-f0-s42-7b--gcwdjm01", + "training_run_name": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-A2-R0_7b_seed42_20260919-040210" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..686339b89c55587ad52ca05b3624fefda43a4e80 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 12000 + +Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_12000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5cd8f01a6471a2b7b5bf53594df4c72f5c0c9443 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 12000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 1536000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D0-A2-R0_7b_seed42_20260919-040210/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 3145728000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e7d37370d4656a0186c1babab74301340c3fc58d --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 12000, + "files": { + "README.md": { + "sha256": "2be766389298f20f09c07bd2b7768dcdcbeafb35203c6350e51b0d285a5d00fb", + "size": 2770 + }, + "config.json": { + "sha256": "e648b3b97cff50579a9667c4614d82cf56eaa46e3ecc30ca5327dc64ffb4570d", + "size": 7254 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "85c568771555459f86c50cf19171dbe9cf501de7b2f0baa97fc3390f98846cd5", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "c7a5b198d7563bb8f6d44f685254d48749a0bcc7fc09fe61b05990ad14442e90", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_12000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eef4a1a98b8fb80c932759c4b6b432fa81527c9e --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85c568771555459f86c50cf19171dbe9cf501de7b2f0baa97fc3390f98846cd5 +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8b8e97930a09985d07e73b4e1517df9e40b453 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_12000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7b4b071261db69176363faeca90eb312063a8047 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 16000 + +Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_16000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..97e91a33fc921fc193a9819ecea2c399e2a29abf --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 16000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 2048000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D0-A2-R0_7b_seed42_20260919-040210/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 4194304000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..e1adde6c26374de4a764d2d47b44348b58615d50 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 16000, + "files": { + "README.md": { + "sha256": "71eb2060ff383647ff254e3922cb7127fa9860ae5bc9420f40253234943c6c2d", + "size": 2770 + }, + "config.json": { + "sha256": "49dde2ce8b251fc5bfcd94718596ae60bffb7ad09ba97b841f9f88c50e121320", + "size": 7254 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "c0d471717736b29b5d9306284929fe32ab8e523d3adaf9fa5f936455d420128f", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "c7a5b198d7563bb8f6d44f685254d48749a0bcc7fc09fe61b05990ad14442e90", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_16000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dba15fe16cd69010f4d73209b1198c28a1f3630d --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d471717736b29b5d9306284929fe32ab8e523d3adaf9fa5f936455d420128f +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8b8e97930a09985d07e73b4e1517df9e40b453 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_16000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..88b91dc705ec2ebc8d6527f5a8ce17e751d03247 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 20000 + +Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_20000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..063a2335cdb51872736b58ea9cffc3c3e0a3e1b6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 20000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2560000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,resume,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-A2-R0_7b_seed42_20260919-040210", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 5242880000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4a09c4ca4c266716e3fd113a0101cb6865322412 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 20000, + "files": { + "README.md": { + "sha256": "43523d318016c829caa3cf150361021b1e3c968808716ff38a554f538df0b0a9", + "size": 2770 + }, + "config.json": { + "sha256": "c2ca9c7aa680114af1fbfcd256baf02f5b343e7491b5d9556d47fd9fd35bef09", + "size": 7200 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "b078bbca103fa1922efd3df063dc6ab7870d73867837b499ff72caf919ec6650", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "2b23f209c0a429772d0d8bf5fd467295425460ba99877d969d52945e17e61eee", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_20000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4913f091c5a66ca9a39dd929bf216a4febce1845 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b078bbca103fa1922efd3df063dc6ab7870d73867837b499ff72caf919ec6650 +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..89ca4ef9cae7bd9d8f26d529b63d5eaff0f0266b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_20000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ee02b10c755b140fd3690fdcd130c1c7b93e05d --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 24000 + +Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_24000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9d2f75251bc342dc556935cd1ae2a0655dbfc002 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 24000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3072000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,resume,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-A2-R0_7b_seed42_20260919-040210", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 6291456000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..89c3625c298aff611efd0e335c9f67f5ada00c4c --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 24000, + "files": { + "README.md": { + "sha256": "37d6a0990541689e6648b463d00ba6d12f552fa2e68e029665cfde157931fe43", + "size": 2770 + }, + "config.json": { + "sha256": "2b5b7703f8df654ce24fc646f4b578e0f31444c5e50e83b4cdde96bfac57941e", + "size": 7200 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "50b1f89a9c8ec867cb357ef218b636348dc8035fa1275724547845d4e3cd4b0d", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "2b23f209c0a429772d0d8bf5fd467295425460ba99877d969d52945e17e61eee", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_24000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0c1d01dbd2e69ce6c18d9ceb85e03e8e9e475d86 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b1f89a9c8ec867cb357ef218b636348dc8035fa1275724547845d4e3cd4b0d +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..89ca4ef9cae7bd9d8f26d529b63d5eaff0f0266b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_24000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..db5cb718b991b833c059be9353001e6d1789fa04 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 26703 + +Final saved checkpoint after **7,000,031,232 training input tokens**. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `3763dea5b6c0cb4496405eff8d7a83f103193769`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_26703") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available. Training and final checkpoint save completed; no final evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a0219b1fa58e887a489de4682f4c083202c2dc75 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/config.json @@ -0,0 +1,193 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 26703, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3417984, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": "26703", + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": "0.95", + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": "0.0005", + "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", + "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", + "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", + "GLOBAL_BATCH_SIZE": "128", + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": "false", + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CHECKPOINT_REFINER": "false", + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": "0", + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": "1", + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": "0.2", + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": "3", + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": "1", + "LOOKAHEAD_PSEUDO_TOP_K": "128", + "LOOKAHEAD_REFINER_FFN_SIZE": "5632", + "LOOKAHEAD_REFINER_GATE_BIAS": "-2", + "LOOKAHEAD_REFINER_VARIANT": "R0", + "MICRO_BATCH_SIZE": "8", + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SEED": "42", + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": "26703", + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,resume,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 7000031232, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d38f68a1a452a1381fc9f826a017057d4acba6c9 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 26703, + "files": { + "README.md": { + "sha256": "d33fb8e1c002cdf3bd4cc628b063ca5cc0dfbf618f680fcb3619437341cabfb0", + "size": 2729 + }, + "config.json": { + "sha256": "f77db439a654ffa26e3a2d2c8e8a700087b2f117e3496a8cda81633e626d25dc", + "size": 6786 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "fac8715a56d7cf95671147e51001708e74379b542714241fe2e58b2911ad1cc0", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "2b23f209c0a429772d0d8bf5fd467295425460ba99877d969d52945e17e61eee", + "size": 4352 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_26703", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9bd0e5c20f5d85b0bf9da51fe9396a6fdb97a062 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fac8715a56d7cf95671147e51001708e74379b542714241fe2e58b2911ad1cc0 +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..89ca4ef9cae7bd9d8f26d529b63d5eaff0f0266b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_26703/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..82de04089c02a156a6c0a9a7eeec25e8d3c3213a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 4000 + +Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_4000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a325b49b57ae64e39ef945932114af516268fd17 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 4000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 512000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004158_previous-checkpoints-hf/metadata/r3_D0-A2-R0_7b_seed42_20260919-040210/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 1048576000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4ed2f73277def24178a16ef18956de5c1607ee04 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 4000, + "files": { + "README.md": { + "sha256": "4b23d96d99d957a73d444346dae02701878168ab7ce7fe54934f628bdf650a73", + "size": 2767 + }, + "config.json": { + "sha256": "371e58a9512e38c8bb6ed3fcf167d8a8cbfa5f118fd3420e30f216fa721f94f2", + "size": 7246 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "477c712a70d0957cadd175ed1db18d117db6fba0d19c48a4c79c6b59579823b7", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "c7a5b198d7563bb8f6d44f685254d48749a0bcc7fc09fe61b05990ad14442e90", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_4000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a5eacdf014f33554419e81a3914cda717d47f9d1 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:477c712a70d0957cadd175ed1db18d117db6fba0d19c48a4c79c6b59579823b7 +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8b8e97930a09985d07e73b4e1517df9e40b453 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_4000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/README.md b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..16dd155e168dfc70e91d35e1e32520e41798e61f --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/README.md @@ -0,0 +1,37 @@ +# r3_D0-A2-R0_7b_seed42_20260919-040210 — step 8000 + +Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **659,332,608 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `0ebebec4193a0afebbbfa50a93792cb6f30e7e21`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_8000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/gcwdjm01). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/config.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5c8f9d9103b3799b21c3648e20a461d9f366c6ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/config.json @@ -0,0 +1,202 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": true, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.449999999999999, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 8000, + "checkpoint_refiner": true, + "checkpoint_version": 3.0, + "consumed_train_samples": 1024000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": true, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.2, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": true, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R0", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_fresh_7b_ablation_20260919", + "WANDB_RUN_ID": "gcwdjm01", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r0" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": true, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.2, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R0" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/exports/20260920_three_arms_history/20260920-004521_previous-checkpoints-hf-retry/metadata/r3_D0-A2-R0_7b_seed42_20260919-040210/early", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_next_token", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 93188480, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 659332608, + "total_trainable": 659332608 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refiner_activation": "silu_gated", + "refiner_variant": "R0", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 193, + "tensor_dtypes": { + "torch.bfloat16": 142, + "torch.float32": 51 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 659332608, + "total_trainable_parameters": 659332608, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "0ebebec4193a0afebbbfa50a93792cb6f30e7e21", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210", + "training_tokens": 2097152000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..17e11c516c7920333b9590af431934fe6c816367 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b3dd98bb6ecd655e563f2cdbbb23131626dd45d3 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 8000, + "files": { + "README.md": { + "sha256": "66be61bdf664c1fed6de81b05ee9f9dce4f7777678955ab0fbe2f385a7aa2faa", + "size": 2767 + }, + "config.json": { + "sha256": "7c19c1eebdea91f1a30172ea3c92061ff6da684427c67c31d7dfeca33df13c80", + "size": 7253 + }, + "launch_config.env": { + "sha256": "5fa99960fc5ed3f0163c4bb6ab1974ea62270f1db074a7f7793b50ff0637e384", + "size": 386 + }, + "model.safetensors": { + "sha256": "ab124d604a4241b184e14c2588b5e9b4964362233834cadcbc407081e7828068", + "size": 1318831144 + }, + "training_recipe.json": { + "sha256": "c7a5b198d7563bb8f6d44f685254d48749a0bcc7fc09fe61b05990ad14442e90", + "size": 4350 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-A2-R0_7b_seed42_20260919-040210/step_8000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-A2-R0_7b_seed42_20260919-040210" +} diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5c8372473a1528efd6407a98021e80ed36bc6da5 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab124d604a4241b184e14c2588b5e9b4964362233834cadcbc407081e7828068 +size 1318831144 diff --git a/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8b8e97930a09985d07e73b4e1517df9e40b453 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0-P0p2-F0-seed42-7b--gcwdjm01/step_8000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": true, + "checkpoint_refiner": true, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/NAMING.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..668887713980249ac9a36b646e9b352e9a18a0a5 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "djlht18n", + "previous_wandb_name": "r3-a2-r0-d0p2-p0p2-f0-s42-7b--djlht18n", + "training_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "canonical_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R0", + "D": 0.2, + "P": 0.2, + "F": 0, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_d1_a2_r0_7b_seed42_20260919-035203-resume" + ] +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b44fcc00c94709a458c3aed1a80d25b116c5718 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_12000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 12000; consumed 3,145,728,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..665832aff6d7a9983a059d71fb39043f64c66ed9 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203", + "iteration": 12000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": true, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 3145728000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": true, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": true + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b4b4e226e0c58b6245dd239624910bacba90cd0f --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1929, + "sha256": "9b41e52a3d3da6609a5754bea758e463024c945c4d7a4b63b54c2a0f8d492516" + }, + "config.json": { + "size": 3666, + "sha256": "02ceca3ed343a81b06cfb067df32ef6e2d2b65f8e6745683f685921f2cd96c33" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "a73de81a0fdecbf66530afc42cef9455f79e5bfa399d03599c4a9833a442f8cd" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd5422ab04af9d020dce6bafe4279e7b1b59fdc0 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a73de81a0fdecbf66530afc42cef9455f79e5bfa399d03599c4a9833a442f8cd +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_12000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b1cfc2b65936774b845382c6026cdd5f28aac5e5 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_16000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 16000; consumed 4,194,304,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e4005f8a376530b040e0e0473f7c95a06ff68ceb --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203", + "iteration": 16000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": true, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 4194304000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": true, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": true + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..3c2294f31a751f00b341005c1cdaf4437bb2fc94 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1929, + "sha256": "20e61e3d078194051f10afc73b6d3d827ac4059b235383d9dbaa4528d5da7fb8" + }, + "config.json": { + "size": 3666, + "sha256": "fa5bb2bccf3f2edbd1333b4ce087224efdaff28488930005899050e91a9bc375" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "9de6d76bb98c3db8f0f6e503e76d65843bc3a86f538388666bce7cc87c2319ce" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a3583f5347f4b521dcece63834d89b68cfde36f --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9de6d76bb98c3db8f0f6e503e76d65843bc3a86f538388666bce7cc87c2319ce +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_16000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4c6f9a2740fa5e3edc19d50628b05ca77de6b3c5 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_20000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 20000; consumed 5,242,880,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..230049f6c8309172383ae2ece8f7cd1ef2f0f990 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "iteration": 20000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 5242880000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..f9647df031364376a6331a89d22de0de831b7b91 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1929, + "sha256": "d00794cf61d025443ff6ef96034adfdb3df1aacf135edb6939b2f58130044547" + }, + "config.json": { + "size": 3677, + "sha256": "2f7a763922f4d0dd450738911bf8dcbd10d12bffd70c21dd9b6d15d1d94c8e28" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "0ca3cdea0e185c05dee0f667f4a4b1087f98da55c36b7a5f7a92b7056f5dcf89" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..23d754223509bdb584a54eb7350024dad38eef89 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca3cdea0e185c05dee0f667f4a4b1087f98da55c36b7a5f7a92b7056f5dcf89 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_20000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..15928d1ccac9ec2917e4371f077239b317a9eb7c --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_24000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 24000; consumed 6,291,456,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..556c0ff24d9a692159184702a793aed12d2c3df6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "iteration": 24000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 6291456000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b6f3ee07ecb69f0630d6c8babf2a35597ee64d0f --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1929, + "sha256": "440c689a8db0e903aa3067733f0698d020dc1fabad72a3e4c7c7b06a68868c73" + }, + "config.json": { + "size": 3677, + "sha256": "56eac3b2ffd0d23edccead2539a5fef3011e2453e195004be358f06fc5968b8d" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "161c8413019fc55253b1048028bbd263bae75af9b00d2e1760210fcf6cd67fdd" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ba62481639adce1a7316fd8547ecab7139d7fc16 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:161c8413019fc55253b1048028bbd263bae75af9b00d2e1760210fcf6cd67fdd +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_24000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3a758c7b40244875a4560b123eaf68555c6075f1 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_26703 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Training reached step 26,703 / 7,000,031,232 input tokens; the source log confirms checkpoint save and program exit at the configured budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..393f5e05203a168a50e78d3d71fc3a5c80bb1fbd --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/config.json @@ -0,0 +1,97 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "iteration": 26703, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": false, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 7000031232, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": false, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": false + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Successfully saved step 26703 and exited at configured budget, per node-0 log. No final-validation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..38a40e28cd16c3cf2cb2449e545f1210724e72fd --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1937, + "sha256": "1eb746b20df05819ff1330872bb92f8fb564af538ddb92e91c6e2535da4e2c52" + }, + "config.json": { + "size": 3606, + "sha256": "66d77305551c604596fb144fa0a4d57a8735cfb3f6275918641ea616c6baa3e3" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "34786ede07c1aba27bf0782c39e01d817c7315d5202341f53ae34155cc745d4f" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0fd0d1fb6b48956262c385cf49be85b586d56cd --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34786ede07c1aba27bf0782c39e01d817c7315d5202341f53ae34155cc745d4f +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_26703/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..799973ac43db03a127bd2a2f5b706d2c56ecfc34 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_4000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 4000; consumed 1,048,576,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f76b26ebaf1f1fcaf73d88f48e0d2438ab23b6d1 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203", + "iteration": 4000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": true, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 1048576000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": true, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": true + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..83dec644d37a491b2abab304d4809d81f7878dbb --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1927, + "sha256": "c20fcc0b37214dad0ee4fc500bc6bf04fa070404b64a710e95d352f62d5ef937" + }, + "config.json": { + "size": 3665, + "sha256": "73b3dc7e05b59dccacd64e816d2508cbf61b31f0546b0315ad5c78aec99bc42d" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "30b034be5c3bee4f1082168b0c3d2612d90408a51e901b06e91ca2905464da39" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..49ce1daf12bed2cd8219f0b18ed7dccb77cf4dc4 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30b034be5c3bee4f1082168b0c3d2612d90408a51e901b06e91ca2905464da39 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_4000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/README.md b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e7d9bd4b52e84eba8c173aba6e1b37e47c3d72bc --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/README.md @@ -0,0 +1,24 @@ +# r3_d1_a2_r0_7b_seed42_20260919-035203-resume / step_8000 + +Native Lookahead D1 A2 R0 weights, 659,332,608 parameters. Intermediate checkpoint at step 8000; consumed 2,097,152,000 input tokens. The full run subsequently reached its 26,703-step budget. No final validation result is asserted. + +Training code: https://github.com/FFishy-git/loopTF-workspace/tree/a8992d1547b32b53044800d836a6c36157df3737 (repository access may be required). The backbone source revision in config.json is distinct from this training revision. Install the project's dependencies at the training revision. + +This is a native state-dict safetensors export, not a Transformers AutoModel package or a full training-resume bundle. Optimizer, scheduler, RNG and dataloader states are excluded. K3 includes the seed pass. Effective D1 draft loss=0.2, future loss=0.2, A2 top-k=128, R0; constructor options are pinned in config.json. The base launch snapshot references common.env; use the resolved JSON for loading. + +```python +import json, torch +from safetensors.torch import load_file +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.policy import LookaheadConfig +from lookahead_tf.hf.model import LookaheadTransformer +c = json.load(open("config.json")) +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**c["model_config"]), LookaheadConfig(**c["lookahead_config"])) +weights = load_file("model.safetensors") +assert all(k.startswith("model.") for k in weights) +model.load_state_dict({k.removeprefix("model."): v for k, v in weights.items()}, strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer with the added assistant-end token described in config.json. Mixed saved dtypes are preserved with assign=True. All tensor names, shapes, dtypes and values were compared to the source, checked finite, and strictly loaded on CPU before upload. manifest.json hashes every payload file except itself. diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/config.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5b145df977769027b638482c474d6e53b61d3f99 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/config.json @@ -0,0 +1,98 @@ +{ + "checkpoint_format": "safetensors_native_megatron_state_dict", + "architecture": "lookahead", + "backbone_profile": "paper2604_vanilla", + "run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203-resume", + "checkpoint_saved_run_name": "r3_d1_a2_r0_7b_seed42_20260919-035203", + "iteration": 8000, + "checkpoint_version": 3.0, + "training_code_revision": "a8992d1547b32b53044800d836a6c36157df3737", + "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", + "parameter_counts": { + "total": 659332608, + "trainable": 659332608 + }, + "model_config": { + "model_type": "full_bandwidth", + "pretrained_model_name_or_path": null, + "revision": null, + "trust_remote_code": false, + "attn_implementation": "sdpa", + "vocab_size": 32001, + "hidden_size": 1408, + "intermediate_size": 5632, + "num_hidden_layers": 20, + "num_attention_heads": 11, + "num_key_value_heads": 11, + "max_position_embeddings": 2048, + "paper_sliding_window": 2048, + "paper_full_attention_interval": 6, + "rope_theta": 10000, + "rms_norm_eps": 1e-06, + "initializer_range": 0.02, + "attention_dropout": 0.0, + "tie_word_embeddings": false, + "gradient_checkpointing": true, + "hf_config_overrides": {}, + "backbone_profile": "paper2604_vanilla", + "padded_vocab_size": 32064 + }, + "lookahead_config": { + "mode": "parallel", + "num_passes": 3, + "feedback_loss_weight": 1.0, + "attention_backend": "sdpa_auto", + "checkpoint_refiner": true, + "cache_layer_indices": null, + "refiner_intermediate_size": 5632, + "refiner_gate_bias": -2.0, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + }, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "precision": "mixed bfloat16 and float32; saved tensor dtypes preserved", + "training_tokens": 2097152000, + "training_budget_tokens": 7000031232, + "tokenizer": { + "repository": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "added_tokens": [ + "<|assistant_end|>" + ], + "active_vocab_size": 32001, + "padded_vocab_size": 32064 + }, + "effective_overrides": { + "lookahead_mode": "parallel", + "lookahead_num_passes": 3, + "lookahead_feedback_loss_weight": 1.0, + "lookahead_attention_backend": "sdpa_auto", + "lookahead_checkpoint_refiner": true, + "lookahead_cache_layers": null, + "lookahead_refiner_ffn_size": 5632, + "lookahead_refiner_gate_bias": -2.0, + "lookahead_future_loss_weight": 0.2, + "lookahead_future_gradient_mode": "all", + "lookahead_allow_objective_change_on_resume": false, + "lookahead_draft_loss_weight": 0.2, + "lookahead_pseudo_initializer": "A2", + "lookahead_pseudo_top_k": 128, + "lookahead_pseudo_temperature": 1.0, + "lookahead_refiner_variant": "R0", + "lookahead_sliding_window": 2048, + "lookahead_full_attention_interval": 6, + "lookahead_z_loss_coefficient": 0.0, + "lookahead_gradient_checkpointing": true + }, + "launch_config_note": "Original base snapshot sources common.env; authoritative resolved architecture is recorded in model_config and lookahead_config; do not execute snapshot standalone.", + "non_tensor_state": {}, + "training_completion": "Intermediate checkpoint from a run that later reached its configured budget. No evaluation metric asserted.", + "loading": "Use pinned project revision, remove exactly model. prefix and load native LookaheadTransformer with strict=True, assign=True." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/launch_config.env b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..7fc407087a3b8b66b5bc801e0cc2d048e6ed6ab6 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/launch_config.env @@ -0,0 +1,9 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d1-a2-r0 +LOOKAHEAD_DRAFT_LOSS_WEIGHT=.2 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R0 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d1-a2-r0 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/manifest.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..31120bc37826e5207b19c2e0877d20163f2ed7f0 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/manifest.json @@ -0,0 +1,25 @@ +{ + "files": { + "README.md": { + "size": 1927, + "sha256": "a167c1cad4d044d4fd88e622696374c37412448fe25a9e9fa2b00a331d589a8b" + }, + "config.json": { + "size": 3665, + "sha256": "4f5216b3b0535cad8b289e1bee10a39887244121772f04eed11d3262f6ab964c" + }, + "launch_config.env": { + "size": 387, + "sha256": "a8548ec79fcd2382440d86a92bc4d2e7874e49c316f645cb487bcc7bb62851e4" + }, + "model.safetensors": { + "size": 1318830968, + "sha256": "eb2276fa5e5791b370ac9f282d1c4a07b184172e0bc0e903debeec25b919d598" + }, + "training_recipe.json": { + "size": 4352, + "sha256": "accc73cde0f5b0c4bfa0fcc9783b1965a4a08daa0381bc6d163326756e4b4eb7" + } + }, + "note": "Manifest does not hash itself." +} diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/model.safetensors b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76f6930fb0cefae68d4556878916442f127559d2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2276fa5e5791b370ac9f282d1c4a07b184172e0bc0e903debeec25b919d598 +size 1318830968 diff --git a/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/training_recipe.json b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..1805bdc41c95063e07dd04153886e13249ff14f2 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-7b--djlht18n/step_8000/training_recipe.json @@ -0,0 +1,150 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 659332608, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 93188480, + "total_parameters": 659332608 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_next_token", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.449999999999999, + "future_loss_weight": 0.2, + "future_gradient_mode": "all", + "draft_loss_weight": 0.2, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R0", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/NAMING.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/NAMING.json new file mode 100644 index 0000000000000000000000000000000000000000..faf9826f5eece77fe285ff697fd7927c2811bf2b --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/NAMING.json @@ -0,0 +1,21 @@ +{ + "wandb_id": "fob3gqiq", + "previous_wandb_name": "r3-a2-r1-d0-p0-f1-s42-7b--fob3gqiq", + "training_run_name": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "canonical_name": "Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq", + "architecture": "lookahead", + "fields": { + "K": 3, + "A": "A2", + "R": "R1", + "D": 0, + "P": 0, + "F": 1, + "normIn": false, + "normOut": false + }, + "seed": 42, + "hf_old_paths": [ + "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" + ] +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3e2338ec99a8fdba5c64fe3243a01ae2418c2b87 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 12000 + +Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_12000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2a909c1d0e5882bf2c5447f2a38ba7c6c7728a90 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 12000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 1536000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 3145728000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b93286ac0f94ecac198881bb115375b01d1bd9ae --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 12000, + "files": { + "README.md": { + "sha256": "bc16fb5e9a42e4be5fe87e27edc9d3812b5daf7ae42a2abb27c43103c7004d70", + "size": 2779 + }, + "config.json": { + "sha256": "26a7df9463a84ce49633c68d7a4a14d7227490f316be217c23776b0f62c716ef", + "size": 7331 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "ebd1c38fc3d2a8ef48155d7647a48cb7553cdb28924c5809c008650d385ca446", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_12000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ed0dc125d259eb529d1aceee93386264400462f8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd1c38fc3d2a8ef48155d7647a48cb7553cdb28924c5809c008650d385ca446 +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_12000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ae60bec26abbf921c0f994831e8252574928208 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 16000 + +Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_16000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0ef46cf11161a4816e5fcf1addcfa415a5a1235d --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 16000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2048000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 4194304000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..723f1ef8edaac063453b16c585b487ae4c376232 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 16000, + "files": { + "README.md": { + "sha256": "941f3d01478679f791786b1b649bfb8e8c716e54da3f51e62c49a382cbbb2f0a", + "size": 2779 + }, + "config.json": { + "sha256": "1d36d8222585f4c7ae29e9f33237845f5697a6084c0d4a2de268cb29d1831af3", + "size": 7331 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "9f05a355bffff33fc3a2211b4ad1d6a9fb6f8aae6c15191ec88406a25ec25ae5", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_16000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e12461c9dd7d9670431cc5d7c77f7be51d5b2416 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f05a355bffff33fc3a2211b4ad1d6a9fb6f8aae6c15191ec88406a25ec25ae5 +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_16000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b60a3610b8c6265bf437f4375f2f8f844d5913a4 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 20000 + +Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_20000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7c70b92e3fa666d29874ddb8c545e705460aafee --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 20000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 2560000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 5242880000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..18b0b4c3e8b85548aba25bf67a0365799a8f8f16 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 20000, + "files": { + "README.md": { + "sha256": "d04dc733c2a8a229a5e5e00960d543d6f637aa25f37f3a4015b0dfd764c6496c", + "size": 2779 + }, + "config.json": { + "sha256": "d33dff39fab2d804ae2bede6bd12582d1dc357bb4199531ddb436021a95dd28a", + "size": 7331 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "cfb2662c21fd806861e5a80b998e31a0c4f3c900e141d050317f9f839a38036e", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_20000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88cba8efaa2930e242dd795af705234ff226d931 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb2662c21fd806861e5a80b998e31a0c4f3c900e141d050317f9f839a38036e +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_20000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bd3d112b36e12f57595dab00a94c73cba82e995a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 24000 + +Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_24000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8853761909271c5b1475895effb06b85c2b83b33 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 24000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3072000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 6291456000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..aa410a34542660835ee743de69d274dcff8c310e --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 24000, + "files": { + "README.md": { + "sha256": "6d50319c8994f32ecc6a43761a9876cd4b843743e34158685406436de4203219", + "size": 2779 + }, + "config.json": { + "sha256": "f0a87143a5018b87a878be1e47b13c62760b11a422f9e84373926021dca5be7e", + "size": 7331 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "3371c4b2d5b329e588ee87b46d6c5f4ed9a932de5e30a6110f90dc0746903885", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_24000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2ee6f8e66fe8495ce0a78531dc765084926d7ac4 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3371c4b2d5b329e588ee87b46d6c5f4ed9a932de5e30a6110f90dc0746903885 +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_24000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dd452ce775690f6f6463151c0b868a0188bcaab0 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 26703 + +Final checkpoint at step **26,703**, after **7,000,031,232 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_26703") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fad1c6bd4ebdc3ebf95548ecf1c9088637d3f73a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 26703, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 3417984, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": true, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 7000031232, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..bec4aab4b3ce5915160b8cca4301a12268e5e175 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 26703, + "files": { + "README.md": { + "sha256": "ff535ff7c70950c5cfffd617257212aaa2d55963b00783ef70c9dd93a9039393", + "size": 2772 + }, + "config.json": { + "sha256": "0791f1554302f9ab379011ee4ed575b96bdd29246c0fef6a6f231505c35f9b74", + "size": 7330 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "948463b520322d1100071274ab8f3dd1b36cd48bdace367385b6bb904b86bc6e", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_26703", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..854240b6895e97aeb78b651e0f547f30d1b1b288 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:948463b520322d1100071274ab8f3dd1b36cd48bdace367385b6bb904b86bc6e +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_26703/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b559e9ccedf5478d24ea57e08c5808f1a7049196 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 4000 + +Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_4000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1757c286f4b4053692152f3b9f9209ed7dba1623 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 4000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 512000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 1048576000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b33897ec73bb14db302e75ff71318997824e19ec --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 4000, + "files": { + "README.md": { + "sha256": "da7f471f238c319c5e19d5148f147210d851b0d31a0e39300e69c9212d622204", + "size": 2776 + }, + "config.json": { + "sha256": "6f92902b1dd63d3665b8fe1eb544aa544a1823ea81481bc91d90db057d044a39", + "size": 7329 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "4e7abc1f2b7e2d16df797438c94484084aa5e91f49b5755ebd5fdacd74d50aba", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_4000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9487baa520a6fd051b3314d6c51e00b9c6706b9 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7abc1f2b7e2d16df797438c94484084aa5e91f49b5755ebd5fdacd74d50aba +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_4000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/README.md b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..73f089d18c4f8014294c38c536a2e4f87027bfb9 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/README.md @@ -0,0 +1,37 @@ +# r3_D0-P0-A2-R1_7b_seed42_20260919-161651 — step 8000 + +Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **665,275,776 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. + +Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. + +Training revision: `03f93c9993ad936db9d39bcacd2a268e899e09f1`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/03f93c9993ad936db9d39bcacd2a268e899e09f1) with `src` on `PYTHONPATH` and its dependencies installed. + +Remove exactly the leading `model.` prefix to load the native LookaheadTransformer. K3 means three complete passes including the seed. The constructor settings preserve this run’s exact draft loss, future loss, pseudo initializer and refiner options. + +## Loading + +```python +import json +from pathlib import Path +import torch +from safetensors.torch import load_file +export_dir = Path("/path/to/step_8000") +config = json.loads((export_dir / "config.json").read_text()) +state = load_file(str(export_dir / "model.safetensors"), device="cpu") +from lookahead_tf.hf.config import ModelConfig +from lookahead_tf.hf.model import LookaheadTransformer +from lookahead_tf.policy import LookaheadConfig +with torch.device("meta"): + model = LookaheadTransformer(ModelConfig(**config["model_config"]), + LookaheadConfig(**config["lookahead_config"])) +assert all(k.startswith("model.") for k in state) +model.load_state_dict({k.removeprefix("model."):v for k,v in state.items()}, + strict=True, assign=True) +model.eval() +``` + +Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. + +No final held-out evaluation receipt is available for this checkpoint; no final-evaluation result is claimed. + +W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/fob3gqiq). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/config.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/config.json new file mode 100644 index 0000000000000000000000000000000000000000..848f032a9dc8355a28e78649a76a395c54a67420 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/config.json @@ -0,0 +1,205 @@ +{ + "activation": "squared_relu", + "activation_checkpointing": false, + "allow_objective_change_on_resume": false, + "attention": "full_causal", + "attention_output_gate": false, + "backbone_profile": "paper2604_vanilla", + "bias": false, + "block_application_multiplier": 6.8999999999999995, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "checkpoint_format": "safetensors_native_megatron_state_dict", + "checkpoint_iteration": 8000, + "checkpoint_refiner": false, + "checkpoint_version": 3.0, + "consumed_train_samples": 1024000, + "crop_padding_before_loss": true, + "draft_loss_weight": 0.0, + "dropout": 0.0, + "effective_overrides": { + "EXIT_INTERVAL": 26703, + "FEEDBACK_EVAL_PASSES": "", + "FEEDBACK_JITTER_SIGMA": "0.02", + "FEEDBACK_LOG_HIDDEN_CHANGE": "false", + "FEEDBACK_LOSS_WEIGHT": "1.0", + "FEEDBACK_NORMUON_BETA2": 0.95, + "FEEDBACK_PASS_MIXTURE": "1:0.75,2:0.22,3:0.03", + "FEEDBACK_PREFIX_MIXIN": "true", + "FEEDBACK_PROGRESSIVE_STAGES": "[{\"start_fraction\":0.0,\"mixture\":{\"1\":1.0}},{\"start_fraction\":0.75,\"mixture\":{\"2\":0.88,\"3\":0.12}}]", + "FEEDBACK_SCALAR_LR": 0.0005, + "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, + "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, + "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, + "GLOBAL_BATCH_SIZE": 128, + "LOOKAHEAD_ALLOW_OBJECTIVE_CHANGE_ON_RESUME": false, + "LOOKAHEAD_ATTENTION_BACKEND": "sdpa_auto", + "LOOKAHEAD_CACHE_LAYERS": null, + "LOOKAHEAD_CHECKPOINT_REFINER": false, + "LOOKAHEAD_DRAFT_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_FEEDBACK_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_FULL_ATTENTION_INTERVAL": 6, + "LOOKAHEAD_FUTURE_GRADIENT_MODE": "all", + "LOOKAHEAD_FUTURE_LOSS_WEIGHT": 0.0, + "LOOKAHEAD_GRADIENT_CHECKPOINTING": false, + "LOOKAHEAD_MODE": "parallel", + "LOOKAHEAD_NUM_PASSES": 3, + "LOOKAHEAD_PSEUDO_INITIALIZER": "A2", + "LOOKAHEAD_PSEUDO_TEMPERATURE": 1.0, + "LOOKAHEAD_PSEUDO_TOP_K": 128, + "LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT": 1.0, + "LOOKAHEAD_REFINER_FFN_SIZE": 5632, + "LOOKAHEAD_REFINER_GATE_BIAS": -2.0, + "LOOKAHEAD_REFINER_VARIANT": "R1", + "LOOKAHEAD_SLIDING_WINDOW": 2048, + "LOOKAHEAD_Z_LOSS_COEFFICIENT": 0.0, + "MICRO_BATCH_SIZE": 8, + "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", + "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", + "ONLINE_DATASET_SUBSET": "sample-100BT", + "ONLINE_SOURCE_SPLIT": "train", + "ONLINE_SPLIT_SEED": "260421106", + "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", + "ONLINE_VALIDATION_FRACTION": "0.001", + "PP": "1", + "PRECISION": "bf16", + "RUN_NAME": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "SAVE_INTERVAL": 4000, + "SEED": 42, + "SEQ_LENGTH": "2048", + "TP": "1", + "TRAIN_ITERS": 26703, + "WANDB_RUN_GROUP": "r3_zero_intermediate_7b_20260919", + "WANDB_RUN_ID": "fob3gqiq", + "WANDB_TAGS": "product,lookahead,ablation,d0-a2-r1" + }, + "execution_architecture": "lookahead", + "feedback_loss_weight": 1.0, + "ffn_hidden_size": 5632, + "final_validation": null, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "head_dim": 128, + "hidden_size": 1408, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "is_final_checkpoint": false, + "loading_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", + "logit_softcap": 15.0, + "lookahead_config": { + "allow_objective_change_on_resume": false, + "attention_backend": "sdpa_auto", + "cache_layer_indices": null, + "checkpoint_refiner": false, + "draft_loss_weight": 0.0, + "feedback_loss_weight": 1.0, + "future_gradient_mode": "all", + "future_loss_weight": 0.0, + "mode": "parallel", + "num_passes": 3, + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "refined_future_loss_weight": 1.0, + "refiner_gate_bias": -2.0, + "refiner_intermediate_size": 5632, + "refiner_variant": "R1" + }, + "lookahead_mode": "parallel", + "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/research/R3_lookahead_improve/r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "model_config": { + "attention_dropout": 0.0, + "attn_implementation": "sdpa", + "backbone_profile": "paper2604_vanilla", + "gradient_checkpointing": false, + "hf_config_overrides": {}, + "hidden_size": 1408, + "initializer_range": 0.02, + "intermediate_size": 5632, + "max_position_embeddings": 2048, + "model_type": "full_bandwidth", + "num_attention_heads": 11, + "num_hidden_layers": 20, + "num_key_value_heads": 11, + "padded_vocab_size": 32064, + "paper_full_attention_interval": 6, + "paper_sliding_window": 2048, + "pretrained_model_name_or_path": null, + "revision": null, + "rms_norm_eps": 1e-06, + "rope_theta": 10000, + "tie_word_embeddings": false, + "trust_remote_code": false, + "vocab_size": 32001 + }, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "non_tensor_state_dict_entries": {}, + "norm_epsilon": 1e-06, + "num_attention_heads": 11, + "num_kv_heads": 11, + "num_layers": 20, + "objective": "lookahead_parallel_final_pair", + "padded_vocab_size": 32064, + "parameter_counts": { + "backbone_total_trainable": 566144128, + "feedback_parameters": 3964928, + "lookahead_parameters": 99131648, + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_parameters": 665275776, + "total_trainable": 665275776 + }, + "pipeline_parallel_size": 1, + "planned_training_tokens": 7000031232, + "projection_dtype": "bfloat16 (production)", + "pseudo_backend": "sdpa_auto", + "pseudo_initializer": "A2", + "pseudo_temperature": 1.0, + "pseudo_top_k": 128, + "qk_norm": "functional_rmsnorm_after_rope", + "refined_future_loss_weight": 1.0, + "refiner_activation": "silu_gated", + "refiner_variant": "R1", + "residual_and_norm_scale_dtype": "float32", + "residual_multiplier": 1.0, + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "sequence_length": 2048, + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "tensor_count": 194, + "tensor_dtypes": { + "torch.bfloat16": 145, + "torch.float32": 49 + }, + "tensor_parallel_size": 1, + "tied_embeddings": false, + "tokenizer": { + "active_vocab_size": 32001, + "additional_token": "<|assistant_end|>", + "repo_id": "NousResearch/Llama-2-7b-hf", + "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "total_parameters": 665275776, + "total_trainable_parameters": 665275776, + "train_passes": 3, + "training_code_diff_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "training_code_dirty": false, + "training_code_revision": "03f93c9993ad936db9d39bcacd2a268e899e09f1", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651", + "training_tokens": 2097152000, + "vocab_size": 32001, + "width_scale": 22 +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/launch_config.env b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/launch_config.env new file mode 100644 index 0000000000000000000000000000000000000000..b134d28345f93cb25af51274001410e3dbc7b9b8 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/launch_config.env @@ -0,0 +1,12 @@ +# Supported design cell, not a scheduled experiment; see ablation-framework.md. +source "$(dirname "${BASH_SOURCE[0]}")/common.env" +RUN_NAME=lookahead_d0-a2-r1 +# Final-slot objective; no direct CE on either pre-refiner latent. +LOOKAHEAD_DRAFT_LOSS_WEIGHT=0 +LOOKAHEAD_FUTURE_LOSS_WEIGHT=0 +LOOKAHEAD_REFINED_FUTURE_LOSS_WEIGHT=1 +LOOKAHEAD_PSEUDO_INITIALIZER=A2 +LOOKAHEAD_REFINER_VARIANT=R1 +# Keep the refiner FFN width identical across R0 and R1. +LOOKAHEAD_REFINER_FFN_SIZE=5632 +WANDB_TAGS=product,lookahead,ablation,d0-a2-r1 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/manifest.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..c88babe934639db0703a7af2a62d44275949c6d3 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/manifest.json @@ -0,0 +1,30 @@ +{ + "checkpoint_iteration": 8000, + "files": { + "README.md": { + "sha256": "7ae7309eb8900af50aef9d30e943e6ae0bfd7bbaa80261b03f79eff48e5318aa", + "size": 2776 + }, + "config.json": { + "sha256": "3d384aee5ac437d84128da2a7e2b1de577614409c4bc4e3cb8a9c3ad3a12c645", + "size": 7330 + }, + "launch_config.env": { + "sha256": "462222782f87efa565da30e051362337478aa563e5210f1da7657bf9bc72a206", + "size": 523 + }, + "model.safetensors": { + "sha256": "a4e1eaf57a2ad8c81e54274c8b15027292fa7a8293d0463a6d017545a494fb63", + "size": 1330711936 + }, + "training_recipe.json": { + "sha256": "df38f204686e2c71752a1c1ad5a23f356fabe500eda028baaf0090e6902afb3f", + "size": 4392 + } + }, + "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", + "path_in_repo": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651/step_8000", + "repo_id": "Siyuc/loopTF-workspace", + "repo_type": "model", + "training_run": "r3_D0-P0-A2-R1_7b_seed42_20260919-161651" +} diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/model.safetensors b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4d521107668108b392efb283ac426aabad637fb9 --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4e1eaf57a2ad8c81e54274c8b15027292fa7a8293d0463a6d017545a494fb63 +size 1330711936 diff --git a/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/training_recipe.json b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/training_recipe.json new file mode 100644 index 0000000000000000000000000000000000000000..edca748b89babef562a5c64696592aa4dee5095a --- /dev/null +++ b/Lookahead-K3-LA2-A2-R1-D0-P0-F1-seed42-7b--fob3gqiq/step_8000/training_recipe.json @@ -0,0 +1,151 @@ +{ + "recipe": "paper2604_21106", + "width": 1408, + "total_tokens": 7000031232, + "batch_tokens": 262144, + "reference_width": 640, + "reference_tokens": 983315200, + "reference_batch_tokens": 262144, + "base_lrs": { + "matrix": 0.014, + "embedding": 0.3, + "unembedding": 0.004, + "norm": 0.005, + "other": 0.005 + }, + "peak_lrs": { + "matrix": 0.007470638533717786, + "embedding": 0.2022599587389726, + "unembedding": 0.0026967994498529684, + "norm": 0.00337099931231621, + "other": 0.00337099931231621 + }, + "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", + "tied_embedding_policy": "unembedding (one AdamW update)", + "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", + "weight_decay": 0.0, + "warmup_steps": 0, + "final_lr_fraction": 0.1, + "optimizer": { + "name": "muonh", + "matrix_lr": 0.014, + "scalar_lr": null, + "matrix_weight_decay": 0.0, + "scalar_weight_decay": 0.0, + "embedding_lr": 0.3, + "unembedding_lr": 0.004, + "norm_lr": 0.005, + "other_lr": 0.005, + "betas": [ + 0.8, + 0.95 + ], + "eps": 1e-10, + "normuon_momentum": 0.95, + "normuon_beta2": 0.95, + "normuon_nesterov": true, + "normuon_adjust_lr": "spectral_norm", + "max_grad_norm": null + }, + "schedule": { + "kind": "linear", + "warmup_steps": 0, + "cooldown_fraction": 0.0, + "cooldown_style": "linear", + "final_lr_fraction": 0.1, + "z_loss_coefficient": 0.0, + "adamc_weight_decay": false + }, + "group_weight_decay": { + "matrix": 0.0, + "embedding": 0.0, + "unembedding": 0.0, + "norm": 0.0, + "other": 0.0 + }, + "recipe_overrides": {}, + "requested_batch_tokens": 262144, + "sequence_length": 2048, + "batch_assumptions": { + "global_batch_tokens": 262144, + "sequence_length": 2048, + "round_up": false + }, + "lr_transfer": "mup_hyperp_released_source", + "assumptions": [], + "backend_kernel_assumptions": {}, + "paper_settings_overridden": false, + "model": { + "backbone_profile": "paper2604_vanilla", + "source_commit": "122366472a31e57982a2877eeee10bb115477713", + "width_scale": 22, + "hidden_size": 1408, + "num_layers": 20, + "head_dim": 128, + "num_attention_heads": 11, + "num_kv_heads": 11, + "ffn_hidden_size": 5632, + "activation": "squared_relu", + "attention": "full_causal", + "attention_output_gate": false, + "qk_norm": "functional_rmsnorm_after_rope", + "norm_epsilon": 1e-06, + "model_level_norms": [ + "embedding", + "recur", + "final" + ], + "rope_base": 10000.0, + "rope_convention": "split_half_negative_angle_bf16_cos_sin", + "bias": false, + "dropout": 0.0, + "residual_multiplier": 1.0, + "residual_and_norm_scale_dtype": "float32", + "projection_dtype": "bfloat16 (production)", + "tied_embeddings": false, + "vocab_size": 32001, + "padded_vocab_size": 32064, + "logit_softcap": 15.0, + "crop_padding_before_loss": true, + "initialization": { + "embedding_normal_std": 1.0, + "head_normal_std": 0.001, + "matrix_uniform_bound": "sqrt(3/fan_in)", + "norm_scale": 1.0 + }, + "parameter_counts": { + "non_embedding_matrix_formula": 475791360, + "non_embedding_paper_N": 475851904, + "total_trainable": 665275776, + "feedback_parameters": 3964928, + "backbone_total_trainable": 566144128, + "lookahead_parameters": 99131648, + "total_parameters": 665275776 + }, + "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", + "execution_architecture": "lookahead", + "objective": "lookahead_parallel_final_pair", + "lookahead_mode": "parallel", + "train_passes": 3, + "feedback_loss_weight": 1.0, + "cache_layer_indices": [ + 0, + 10, + 19 + ], + "refiner_activation": "silu_gated", + "pseudo_backend": "sdpa_auto", + "activation_checkpointing": false, + "checkpoint_refiner": false, + "block_application_multiplier": 6.8999999999999995, + "future_loss_weight": 0.0, + "future_gradient_mode": "all", + "draft_loss_weight": 0.0, + "refined_future_loss_weight": 1.0, + "pseudo_initializer": "A2", + "pseudo_top_k": 128, + "pseudo_temperature": 1.0, + "refiner_variant": "R1", + "allow_objective_change_on_resume": false + } +}