Add Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp step 26703 weights

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/README.md

@@ -0,0 +1,25 @@
+# Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp — step 26703
+
+Final model weights; 671,227,392 parameters; 7,000,031,232 consumed training input tokens.
+
+Native Megatron tensor names, shapes, dtypes and values are preserved. Every tensor was checked for finite values and exact serialization equality; strict native loading passed on CPU. This is a weights-only export, not a training-resume bundle or Transformers AutoModel export.
+
+Use loopTF-workspace loading revision `57b880dff6611944aa58ab587930322694b3c47d`. The tokenizer files are in `tokenizer/`; training settings and any available final validation metrics are in `config.json`. Exporting does not run evaluation.
+
+```python
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from safetensors.torch import load_file
+from loop_transformer_backends.benchmarks.models import build_native
+
+folder = Path('/path/to/downloaded/step_folder')
+config = json.loads((folder / 'config.json').read_text())
+state = load_file(str(folder / 'model.safetensors'), device='cpu')
+model, metadata = build_native({
+    'args': SimpleNamespace(**config['native_loading_args']),
+    'model': state | config['non_tensor_state_dict_entries'],
+    'iteration': config['iteration'],
+})
+model.eval()
+```

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/config.json

@@ -0,0 +1,142 @@
+{
+  "architecture": "lookahead",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_iteration": 26703,
+  "execution_architecture": "lookahead",
+  "final_checkpoint": true,
+  "final_validation": null,
+  "iteration": 26703,
+  "loading_code_revision": "57b880dff6611944aa58ab587930322694b3c47d",
+  "loading_note": "Native loopTF weights; use the project adapter, not Transformers AutoModel. Optimizer, scheduler, RNG and dataloader state are excluded.",
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "auxiliary_loss_passes": "final",
+    "auxiliary_loss_weighting": "sum",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.0,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.0,
+    "mode": "parallel",
+    "normalize_feedback_embedding": false,
+    "normalize_feedback_output": false,
+    "num_passes": 3,
+    "num_refiner_layers": 3,
+    "pseudo_initializer": "A2",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refined_future_loss_weight": 0.0,
+    "refiner_activation": "swiglu",
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_memory": true,
+    "refiner_memory_only": true,
+    "refiner_variant": "R0"
+  },
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_allow_ffn_override": false,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "native_loading_args": {
+    "architecture": "lookahead",
+    "attention_dropout": 0.0,
+    "backbone_profile": "paper2604_vanilla",
+    "context_parallel_size": 1,
+    "ffn_hidden_size": 5632,
+    "hidden_size": 1408,
+    "init_method_std": 0.02,
+    "layernorm_epsilon": 1e-06,
+    "lookahead_a2_embedding_bag": true,
+    "lookahead_allow_objective_change_on_resume": false,
+    "lookahead_attention_backend": "sdpa_auto",
+    "lookahead_auxiliary_loss_passes": "final",
+    "lookahead_auxiliary_loss_weighting": "sum",
+    "lookahead_cache_layers": null,
+    "lookahead_checkpoint_refiner": false,
+    "lookahead_compile_blocks": true,
+    "lookahead_draft_loss_weight": 0.0,
+    "lookahead_feedback_loss_weight": 1.0,
+    "lookahead_full_attention_interval": 6,
+    "lookahead_future_gradient_mode": "all",
+    "lookahead_future_loss_weight": 0.0,
+    "lookahead_gradient_checkpointing": false,
+    "lookahead_mode": "parallel",
+    "lookahead_normalize_feedback_embedding": false,
+    "lookahead_normalize_feedback_output": false,
+    "lookahead_num_passes": 3,
+    "lookahead_pseudo_initializer": "A2",
+    "lookahead_pseudo_temperature": 1.0,
+    "lookahead_pseudo_top_k": 128,
+    "lookahead_refined_future_loss_weight": 0.0,
+    "lookahead_refiner_ffn_size": 5632,
+    "lookahead_refiner_gate_bias": -2.0,
+    "lookahead_refiner_memory": true,
+    "lookahead_refiner_memory_only": true,
+    "lookahead_refiner_variant": "R0",
+    "lookahead_sliding_window": 2048,
+    "lookahead_z_loss_coefficient": 0.0,
+    "num_attention_heads": 11,
+    "num_layers": 20,
+    "num_query_groups": 11,
+    "padded_vocab_size": 32064,
+    "pipeline_model_parallel_size": 1,
+    "rotary_base": 10000,
+    "seq_length": 2048,
+    "tensor_model_parallel_size": 1,
+    "untie_embeddings_and_output_weights": true,
+    "vocab_size": 32001
+  },
+  "non_tensor_state_dict_entries": {},
+  "parameter_counts": {
+    "total_parameters": 671227392,
+    "total_trainable": 671227392
+  },
+  "tensor_count": 199,
+  "tensor_dtypes": {
+    "torch.bfloat16": 148,
+    "torch.float32": 51
+  },
+  "tokenizer": {
+    "active_vocab_size": 32001,
+    "path": "tokenizer"
+  },
+  "training": {
+    "consumed_train_samples": 3417984,
+    "final_checkpoint": true,
+    "global_batch_size": 128,
+    "input_tokens": 7000031232,
+    "iteration": 26703,
+    "planned_input_tokens": 7000031232,
+    "seed": 42,
+    "sequence_length": 2048,
+    "train_iters": 26703
+  },
+  "training_code_revision": "f37eb269641895368bceffe45f0d17e5a99ba719",
+  "training_run": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp",
+  "training_tokens": 7000031232,
+  "wandb_run_id": "9brnbryp"
+}

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/manifest.json

@@ -0,0 +1,33 @@
+{
+  "checkpoint_iteration": 26703,
+  "files": {
+    "README.md": {
+      "sha256": "e934e8fcb7e6e9df1bb74eb9806841c08eef66a2cfdffbfa49ce29b24d28d65d",
+      "size": 1301
+    },
+    "config.json": {
+      "sha256": "86463bc969f388c69ac184fe1fec7604f025c36518f8ecf8c9bc5fde249c9877",
+      "size": 4829
+    },
+    "model.safetensors": {
+      "sha256": "06d23b4724a64234838b8a6dbc0a341fdcd99361e58d6618d5bf605e0acfb2e0",
+      "size": 1342621256
+    },
+    "tokenizer/special_tokens_map.json": {
+      "sha256": "99e1ac71ff7f0e3e60e45169cdfd7e8540b8c234e7146100e60a02c4751553ee",
+      "size": 134
+    },
+    "tokenizer/tokenizer.json": {
+      "sha256": "60c0314d888788ac33de8cb7262be527cffb0af30f0516cc3dd16b2b378bc6cb",
+      "size": 3619206
+    },
+    "tokenizer/tokenizer_config.json": {
+      "sha256": "77f3faab79c4008cb57c4baab3c75b69ef088c5fb15e8f7af59471fe37abc001",
+      "size": 988
+    }
+  },
+  "path_in_repo": "Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703",
+  "repo_id": "Siyuc/loopTF-workspace",
+  "repo_type": "model",
+  "schema_version": 1
+}

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06d23b4724a64234838b8a6dbc0a341fdcd99361e58d6618d5bf605e0acfb2e0
+size 1342621256

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/tokenizer/special_tokens_map.json

@@ -0,0 +1,8 @@
+{
+  "additional_special_tokens": [
+    "<|assistant_end|>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

Lookahead-K3-LA2-A2-R0-D0-P0-F0-seed42-refinerMemoryOnly-7b-compiled-a2bag-MB4--9brnbryp/step_26703/tokenizer/tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<|assistant_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|assistant_end|>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}