Add Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci step 26703

Native weights-only checkpoint export. Source and safetensors were strictly reconstructed and verified locally before upload.

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/README.md

@@ -0,0 +1,7 @@
+# Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci step 26,703
+
+Native weights-only export for Lookahead with causal refiner memory. This checkpoint represents 7,000,031,232 consumed input tokens.
+
+`model.safetensors` contains the native Megatron model state only. It excludes optimizer, scheduler, RNG, dataloader, datasets, caches, logs, and credentials. It is not a Transformers `AutoModel.from_pretrained` checkpoint.
+
+Use the exact training code revision and strict loading instructions in `config.json`. Use `manifest.json` to verify payload hashes.

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/config.json

@@ -0,0 +1,1389 @@
+{
+  "architecture": "lookahead",
+  "backbone_profile": "paper2604_vanilla",
+  "checkpoint_format": "safetensors_native_megatron_state_dict",
+  "checkpoint_version": 3.0,
+  "excluded_null_extra_state_keys": [],
+  "excluded_state": [
+    "optimizer",
+    "opt_param_scheduler",
+    "rng_state",
+    "dataloader"
+  ],
+  "iteration": 26703,
+  "loading": {
+    "format": "native Megatron model state saved with safetensors; not a Transformers AutoModel",
+    "instructions": "Construct lookahead_tf.hf.model.LookaheadTransformer with model_config and lookahead_config at training_code_revision, remove exactly the model. prefix, then call load_state_dict(..., strict=True, assign=True).",
+    "key_prefix": "model."
+  },
+  "lookahead_config": {
+    "allow_objective_change_on_resume": false,
+    "attention_backend": "sdpa_auto",
+    "cache_layer_indices": null,
+    "checkpoint_refiner": false,
+    "draft_loss_weight": 0.2,
+    "feedback_loss_weight": 1.0,
+    "future_gradient_mode": "all",
+    "future_loss_weight": 0.2,
+    "mode": "parallel",
+    "normalize_feedback_embedding": false,
+    "normalize_feedback_output": false,
+    "num_passes": 3,
+    "pseudo_initializer": "A2",
+    "pseudo_temperature": 1.0,
+    "pseudo_top_k": 128,
+    "refined_future_loss_weight": 0.0,
+    "refiner_gate_bias": -2.0,
+    "refiner_intermediate_size": 5632,
+    "refiner_memory": true,
+    "refiner_memory_only": true,
+    "refiner_variant": "R0"
+  },
+  "model_arguments": {
+    "_is_global_batch_size_explicitly_specified": true,
+    "account_for_embedding_in_pipeline_split": false,
+    "account_for_loss_in_pipeline_split": false,
+    "accumulate_allreduce_grads_in_fp32": true,
+    "activation_func_clamp_value": null,
+    "activation_offload_fraction": 1.0,
+    "adam_beta1": 0.8,
+    "adam_beta2": 0.95,
+    "adam_eps": 1e-10,
+    "adaptive_muon_beta2": 0.95,
+    "adaptive_muon_eps": 1e-10,
+    "adaptive_muon_moment2_method": "normuon",
+    "add_bias_linear": false,
+    "add_position_embedding": true,
+    "add_qkv_bias": false,
+    "adlr_autoresume": false,
+    "adlr_autoresume_interval": 1000,
+    "align_grad_reduce": true,
+    "align_param_gather": false,
+    "allow_ambiguous_pad_tokens": false,
+    "app_tag_run_name": null,
+    "app_tag_run_version": "0.0.0",
+    "apply_query_key_layer_scaling": false,
+    "apply_residual_connection_post_layernorm": false,
+    "apply_rope_fusion": false,
+    "apply_wd_to_qk_layernorm": false,
+    "architecture": "lookahead",
+    "architecture_module": [],
+    "async_ckpt_cpu_priority": 10,
+    "async_ckpt_io_priority": 3,
+    "async_ckpt_use_cpu_shm": false,
+    "async_save": false,
+    "async_strategy": "mcore",
+    "attention_backend": "AttnBackend.auto",
+    "attention_cp_layout": "zigzag",
+    "attention_dropout": 0.0,
+    "attention_output_gate": false,
+    "attention_softmax_in_fp32": false,
+    "auto_detect_ckpt_format": false,
+    "backbone_metadata": {
+      "activation": "squared_relu",
+      "activation_checkpointing": false,
+      "allow_objective_change_on_resume": false,
+      "attention": "full_causal",
+      "attention_output_gate": false,
+      "backbone_profile": "paper2604_vanilla",
+      "bias": false,
+      "block_application_multiplier": 6.449999999999999,
+      "cache_layer_indices": [
+        0,
+        10,
+        19
+      ],
+      "checkpoint_refiner": false,
+      "crop_padding_before_loss": true,
+      "draft_loss_weight": 0.2,
+      "dropout": 0.0,
+      "execution_architecture": "lookahead",
+      "feedback_loss_weight": 1.0,
+      "ffn_hidden_size": 5632,
+      "future_gradient_mode": "all",
+      "future_loss_weight": 0.2,
+      "head_dim": 128,
+      "hidden_size": 1408,
+      "initialization": {
+        "embedding_normal_std": 1.0,
+        "head_normal_std": 0.001,
+        "matrix_uniform_bound": "sqrt(3/fan_in)",
+        "norm_scale": 1.0
+      },
+      "logit_softcap": 15.0,
+      "lookahead_mode": "parallel",
+      "model_level_norms": [
+        "embedding",
+        "recur",
+        "final"
+      ],
+      "norm_epsilon": 1e-06,
+      "normalize_feedback_embedding": false,
+      "normalize_feedback_output": false,
+      "num_attention_heads": 11,
+      "num_kv_heads": 11,
+      "num_layers": 20,
+      "objective": "lookahead_parallel_next_token",
+      "padded_vocab_size": 32064,
+      "parameter_counts": {
+        "backbone_total_trainable": 566144128,
+        "feedback_parameters": 3964928,
+        "lookahead_parameters": 105083264,
+        "non_embedding_matrix_formula": 475791360,
+        "non_embedding_paper_N": 475851904,
+        "total_parameters": 671227392,
+        "total_trainable": 671227392
+      },
+      "projection_dtype": "bfloat16 (production)",
+      "pseudo_backend": "sdpa_auto",
+      "pseudo_initializer": "A2",
+      "pseudo_temperature": 1.0,
+      "pseudo_top_k": 128,
+      "qk_norm": "functional_rmsnorm_after_rope",
+      "refined_future_loss_weight": 0.0,
+      "refiner_activation": "silu_gated",
+      "refiner_memory": true,
+      "refiner_memory_only": true,
+      "refiner_variant": "R0",
+      "residual_and_norm_scale_dtype": "float32",
+      "residual_multiplier": 1.0,
+      "rope_base": 10000.0,
+      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+      "tied_embeddings": false,
+      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+      "train_passes": 3,
+      "vocab_size": 32001,
+      "width_scale": 22
+    },
+    "backbone_profile": "paper2604_vanilla",
+    "barrier_with_L1_time": true,
+    "batch_invariant_backend": "te_native",
+    "batch_invariant_collective": "ordered",
+    "batch_invariant_mode": false,
+    "bert_binary_head": true,
+    "bert_embedder_type": "megatron",
+    "bert_load": null,
+    "bf16": true,
+    "bias_dropout_fusion": false,
+    "bias_gelu_fusion": false,
+    "bias_swiglu_fusion": true,
+    "biencoder_projection_dim": 0,
+    "biencoder_shared_query_context_model": false,
+    "block_data_path": null,
+    "cache_mla_latents": false,
+    "calc_ft_timeouts": false,
+    "calculate_per_token_loss": false,
+    "chat_template": null,
+    "check_for_large_grads": false,
+    "check_for_nan_in_loss_and_grad": true,
+    "check_for_spiky_loss": false,
+    "check_weight_hash_across_dp_replicas_interval": null,
+    "ckpt_assume_constant_structure": false,
+    "ckpt_convert_format": null,
+    "ckpt_convert_save": null,
+    "ckpt_convert_update_legacy_dist_opt_format": false,
+    "ckpt_drop_redundant_extra_state": false,
+    "ckpt_format": "torch",
+    "ckpt_fully_parallel_load": false,
+    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
+    "ckpt_fully_parallel_load_per_rank_objects": false,
+    "ckpt_fully_parallel_load_process_group": "dp",
+    "ckpt_fully_parallel_save": true,
+    "ckpt_fully_parallel_save_deprecated": false,
+    "ckpt_fully_parallel_save_process_group": "dp",
+    "ckpt_load_validate_sharding_integrity": true,
+    "ckpt_pg_tensors_cache_create": false,
+    "ckpt_pg_tensors_cache_path": null,
+    "ckpt_step": null,
+    "classes_fraction": 1.0,
+    "clip_grad": 0.0,
+    "clone_scatter_output_in_embedding": true,
+    "config_logger_dir": "",
+    "consumed_train_samples": 3417984,
+    "consumed_valid_samples": 68608,
+    "context_parallel_size": 1,
+    "cp_comm_type": [
+      "p2p"
+    ],
+    "cpu_offloading_num_layers": 0,
+    "cpu_offloading_retain_pinned_cpu_buffers": false,
+    "create_all_gather_group": false,
+    "create_attention_mask_in_dataloader": false,
+    "cross_entropy_fusion_impl": "native",
+    "cross_entropy_loss_fusion": false,
+    "csa_compress_ratios": null,
+    "csa_compress_rotary_base": 40000.0,
+    "csa_dense_mode": false,
+    "csa_window_size": 128,
+    "cuda_graph_impl": "none",
+    "cuda_graph_modules": [],
+    "cuda_graph_warmup_steps": 3,
+    "curr_iteration": 26702,
+    "data_args_path": null,
+    "data_cache_path": null,
+    "data_parallel_random_init": false,
+    "data_parallel_sharding_strategy": "optim_grads_params",
+    "data_parallel_size": 16,
+    "data_path": null,
+    "data_per_class_fraction": 1.0,
+    "data_sharding": true,
+    "dataloader_defer_npy_index_mmap": false,
+    "dataloader_fast_cache_load": false,
+    "dataloader_inter_document_masking": false,
+    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/relaunch/20260921-204652_lookahead-memoryonly-relaunch/dataloader",
+    "dataloader_type": "external",
+    "ddp_average_in_collective": false,
+    "ddp_bucket_size": null,
+    "ddp_num_buckets": null,
+    "ddp_pad_buckets_for_high_nccl_busbw": false,
+    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
+    "ddp_reduce_scatter_with_fp32_accumulation": false,
+    "decode_only_cuda_graphs": false,
+    "decoder_first_pipeline_num_layers": null,
+    "decoder_last_pipeline_num_layers": null,
+    "decoder_num_layers": null,
+    "decoder_seq_length": null,
+    "decoupled_lr": null,
+    "decoupled_min_lr": null,
+    "decrease_batch_size_if_needed": false,
+    "defer_embedding_wgrad_compute": false,
+    "delay_offload_until_cuda_graph": false,
+    "delay_wgrad_compute": false,
+    "delta_offload_bytes_across_pp_ranks": 0,
+    "deprecated_use_mcore_models": false,
+    "deterministic_mode": false,
+    "dino_bottleneck_size": 256,
+    "dino_freeze_last_layer": 1,
+    "dino_head_hidden_size": 2048,
+    "dino_local_crops_number": 10,
+    "dino_local_img_size": 96,
+    "dino_norm_last_layer": false,
+    "dino_teacher_temp": 0.07,
+    "dino_warmup_teacher_temp": 0.04,
+    "dino_warmup_teacher_temp_epochs": 30,
+    "disable_bf16_reduced_precision_matmul": false,
+    "disable_jit_fuser": false,
+    "disable_msc_deprecated": false,
+    "disable_straggler_on_startup": false,
+    "disable_symmetric_registration": false,
+    "dist_ckpt_format_deprecated": null,
+    "dist_ckpt_optim_fully_reshardable": false,
+    "dist_ckpt_save_pre_mcore_014": false,
+    "dist_ckpt_strictness": "assume_ok_unexpected",
+    "dist_ckpt_workers": 1,
+    "distrib_optim_fully_reshardable_mem_efficient": false,
+    "distribute_saved_activations": false,
+    "distributed_backend": "nccl",
+    "distributed_timeout_minutes": 10,
+    "distributed_timeout_seconds_after_init": null,
+    "do_test": 0,
+    "do_train": 1,
+    "do_valid": 1,
+    "dsa_indexer_head_dim": null,
+    "dsa_indexer_k_norm_epsilon": null,
+    "dsa_indexer_k_norm_fp32": false,
+    "dsa_indexer_loss_coeff": null,
+    "dsa_indexer_n_heads": null,
+    "dsa_indexer_rope_interleaved": false,
+    "dsa_indexer_rotate_activation": true,
+    "dsa_indexer_scoring_relu": true,
+    "dsa_indexer_skip_topk_offset": 0,
+    "dsa_indexer_topk": null,
+    "dsa_indexer_topk_freq": 1,
+    "dsa_indexer_use_sparse_loss": false,
+    "dsa_kernel_backend": "none",
+    "dump_param_to_param_group_map": null,
+    "embedding_init_method_std": null,
+    "embedding_path": null,
+    "empty_unused_memory_level": 0,
+    "enable_chunked_prefill": false,
+    "enable_cuda_graph": false,
+    "enable_experimental": false,
+    "enable_ft_package": false,
+    "enable_full_sharding_in_hsdp": false,
+    "enable_mhc_connections": false,
+    "enable_msc": false,
+    "enable_one_logger": true,
+    "encoder_num_layers": 20,
+    "encoder_seq_length": 2048,
+    "end_weight_decay": 0.0,
+    "eod_mask_loss": false,
+    "ep_overlap_early_attn_memory_release": false,
+    "error_injection_rate": 0,
+    "error_injection_type": "transient_error",
+    "eval_at_start": true,
+    "eval_global_batch_size": 128,
+    "eval_interval": 100,
+    "eval_iters": 2,
+    "eval_micro_batch_size": 8,
+    "evaluation_metadata": {
+      "eval_at_start": true,
+      "final_validation_input_tokens": 70778880,
+      "final_validation_iters": 270,
+      "final_validation_sequences": 34560,
+      "global_batch_size": 128,
+      "initial_iters": 2,
+      "intermediate_input_tokens": 524288,
+      "intermediate_iters": 2,
+      "sequence_length": 2048,
+      "test_iters": 0,
+      "validation_iterator": "continues_existing_held_out_iterator"
+    },
+    "evidence_data_path": null,
+    "exit_duration_in_mins": null,
+    "exit_interval": null,
+    "exit_on_missing_checkpoint": false,
+    "exit_signal": 15,
+    "exit_signal_handler": false,
+    "exit_signal_handler_for_dataloader": false,
+    "exit_signal_handler_for_training": false,
+    "exp_avg_dtype": "torch.float32",
+    "exp_avg_sq_dtype": "torch.float32",
+    "experimental_attention_variant": null,
+    "expert_data_parallel_sharding_strategy": null,
+    "expert_gtp_weight_remat_size": 1,
+    "expert_model_parallel_size": 1,
+    "expert_tensor_parallel_num_weight_shards": 1,
+    "expert_tensor_parallel_size": 1,
+    "external_cuda_graph": false,
+    "fake_process_group": false,
+    "fault_injector_delay_start_iteration": null,
+    "fault_injector_fault_delay": null,
+    "fault_injector_fault_probabilities": null,
+    "fault_injector_fault_types": null,
+    "fault_injector_mtti_seconds": null,
+    "fault_injector_num_ranks": null,
+    "fault_injector_offset_seconds": null,
+    "fault_injector_ranks": null,
+    "fault_injector_seed": null,
+    "feedback_normuon_beta2": 0.95,
+    "feedback_scalar_lr": 0.0005,
+    "feedback_scalar_weight_decay": 0.0,
+    "feedback_z_loss_coefficient": 0.0,
+    "feedback_z_loss_start_fraction": 1.0,
+    "ffn_hidden_size": 5632,
+    "fim_data": false,
+    "fim_eod_token": "<|endoftext|>",
+    "fim_fragment_rate": null,
+    "fim_middle_token": "<fim_middle>",
+    "fim_no_prefix": null,
+    "fim_pad_token": "<fim_pad>",
+    "fim_prefix_token": "<fim_prefix>",
+    "fim_rate": 0.5,
+    "fim_split_sample": null,
+    "fim_spm_rate": 0.5,
+    "fim_suffix_token": "<fim_suffix>",
+    "final_eval_iters": 270,
+    "fine_grained_activation_offloading": false,
+    "fine_grained_offloading_max_inflight_offloads": null,
+    "finetune": false,
+    "first_last_layers_bf16": false,
+    "flash_attention_version": null,
+    "flash_decode": false,
+    "flight_recorder_dump_on_timeout": true,
+    "flight_recorder_dump_path": null,
+    "flight_recorder_extra_dump_on_exec": true,
+    "flight_recorder_include_only_active": true,
+    "flight_recorder_include_stack_trace": false,
+    "flight_recorder_trace_buffer_size": 2000,
+    "fp16": false,
+    "fp16_lm_cross_entropy": false,
+    "fp32_residual_connection": false,
+    "fp4": null,
+    "fp4_param_gather": false,
+    "fp4_quantizer_factory": null,
+    "fp4_recipe": "nvfp4",
+    "fp8": null,
+    "fp8_amax_compute_algo": "most_recent",
+    "fp8_amax_history_len": 1,
+    "fp8_interval": 1,
+    "fp8_margin": 0,
+    "fp8_output_proj": false,
+    "fp8_param_gather": false,
+    "fp8_quantizer_factory": null,
+    "fp8_recipe": "delayed",
+    "fp8_wgrad": true,
+    "freeze_all_layers": false,
+    "fsdp_db_use_persist_buf_on_alloc_fail": false,
+    "fsdp_double_buffer": false,
+    "fsdp_manual_registration": false,
+    "ft_num_warmup_iters": 5,
+    "full_validation": false,
+    "fused_residual_rmsnorm": false,
+    "gdp_cutedsl_kernel": false,
+    "gdp_num_chunk_states_to_recompute": 2,
+    "gdp_num_householder": 3,
+    "global_batch_size": 128,
+    "glu_linear_offset": 0.0,
+    "gpu_sniff_test_interval": null,
+    "grad_reduce_in_bf16": false,
+    "gradient_accumulation_fusion": false,
+    "gradient_reduce_div_fusion": true,
+    "group_query_attention": false,
+    "grpo_clamp_eps_lower": 0.01,
+    "grpo_clamp_eps_upper": 0.01,
+    "grpo_entropy_term_weight": 0.0,
+    "grpo_filter_groups_with_same_reward": false,
+    "grpo_group_size": 2,
+    "grpo_iterations": 2,
+    "grpo_kl_beta": 0.001,
+    "grpo_prompts_per_step": 32,
+    "gtp_expert_remat_nccl_ub": false,
+    "gtp_remat_nccl_ub": false,
+    "gtp_remat_opt_in_modules": [],
+    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
+    "gtp_weight_remat_size": 1,
+    "head_lr_mult": 1.0,
+    "heterogeneous_layers_config_encoded_json": null,
+    "heterogeneous_layers_config_path": null,
+    "hfsdp_param_gather_overlap": false,
+    "hidden_dropout": 0.0,
+    "hidden_size": 1408,
+    "hierarchical_context_parallel_sizes": null,
+    "high_priority_a2a_comm_stream": false,
+    "high_priority_stream_groups": [],
+    "hybrid_context_parallel": false,
+    "hybrid_layer_pattern": null,
+    "hybrid_override_pattern": null,
+    "hysteresis": 2,
+    "ict_head_size": null,
+    "ict_load": null,
+    "img_h": 224,
+    "img_w": 224,
+    "indexer_batch_size": 128,
+    "indexer_log_interval": 1000,
+    "inference_batch_times_seqlen_threshold": -1,
+    "inference_coordinator_port": null,
+    "inference_cuda_graph_all_prefills": false,
+    "inference_cuda_graph_max_tokens": 512,
+    "inference_cuda_graph_scope": "InferenceCudaGraphScope.none",
+    "inference_disable_ep_consensus": false,
+    "inference_disable_triton_nvls_kernels": false,
+    "inference_dynamic_batching": false,
+    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
+    "inference_dynamic_batching_async_sched_mode": "async",
+    "inference_dynamic_batching_block_size": 256,
+    "inference_dynamic_batching_buffer_size_gb": 40.0,
+    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
+    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
+    "inference_dynamic_batching_enable_prefix_caching": false,
+    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
+    "inference_dynamic_batching_mamba_memory_ratio": null,
+    "inference_dynamic_batching_max_requests": null,
+    "inference_dynamic_batching_max_tokens": null,
+    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
+    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
+    "inference_dynamic_batching_num_cuda_graphs": 16,
+    "inference_dynamic_batching_paused_buffer_size_gb": null,
+    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
+    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
+    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
+    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
+    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
+    "inference_dynamic_batching_sampling_backend": "torch",
+    "inference_dynamic_batching_track_generated_token_events": false,
+    "inference_dynamic_batching_track_paused_request_events": false,
+    "inference_dynamic_batching_unified_memory_level": 0,
+    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
+    "inference_flashinfer_mxfp8_token_capacity": null,
+    "inference_fuse_tp_communication": false,
+    "inference_grouped_gemm_backend": "vllm",
+    "inference_logging_step_interval": 0,
+    "inference_max_requests": 8,
+    "inference_max_seq_length": 2560,
+    "inference_moe_disable_fused_quant_kernels": false,
+    "inference_moe_token_dispatcher_type": "nvls",
+    "inference_rng_tracker": false,
+    "inference_shards": null,
+    "inference_text_gen_server_logging": false,
+    "inference_use_synchronous_zmq_collectives": false,
+    "inference_wandb_logging": false,
+    "init_method_std": 0.02,
+    "init_method_xavier_uniform": false,
+    "init_model_with_meta_device": false,
+    "initial_loss_scale": 4294967296,
+    "inprocess_active_world_size": 16,
+    "inprocess_barrier_timeout": 120,
+    "inprocess_completion_timeout": 120,
+    "inprocess_empty_cuda_cache": false,
+    "inprocess_granularity": "node",
+    "inprocess_hard_timeout": 90,
+    "inprocess_heartbeat_interval": 30,
+    "inprocess_heartbeat_timeout": 60,
+    "inprocess_last_call_wait": 1,
+    "inprocess_max_iterations": null,
+    "inprocess_monitor_process_interval": 1.0,
+    "inprocess_monitor_thread_interval": 1.0,
+    "inprocess_progress_watchdog_interval": 1.0,
+    "inprocess_restart": false,
+    "inprocess_soft_timeout": 60,
+    "inprocess_termination_grace_time": 1,
+    "is_hybrid_model": false,
+    "iter_per_epoch": 1250,
+    "iteration": 0,
+    "iterations_to_skip": [],
+    "keep_fp8_transpose_cache": false,
+    "kitchen_attention_backend": "sdpa",
+    "kv_channels": 128,
+    "kv_lora_rank": 32,
+    "langrl_env_config": null,
+    "layernorm_epsilon": 1e-06,
+    "layernorm_zero_centered_gamma": false,
+    "lazy_mpu_init": false,
+    "linear_attention_freq": null,
+    "linear_conv_kernel_dim": 4,
+    "linear_cp_layout": "zigzag",
+    "linear_cp_mode": "headwise",
+    "linear_key_head_dim": 128,
+    "linear_num_key_heads": 16,
+    "linear_num_value_heads": 32,
+    "linear_value_head_dim": 128,
+    "lion_beta1": 0.95,
+    "lion_beta2": 0.98,
+    "load": null,
+    "load_main_params_from_ckpt": false,
+    "local_rank": 0,
+    "log_device_memory_used": false,
+    "log_energy": false,
+    "log_interval": 1,
+    "log_loss_scale_to_tensorboard": true,
+    "log_max_attention_logit": false,
+    "log_memory_interval": null,
+    "log_memory_to_tensorboard": false,
+    "log_num_zeros_in_grad": false,
+    "log_params_norm": false,
+    "log_progress": false,
+    "log_straggler": false,
+    "log_throughput": false,
+    "log_timers_to_tensorboard": true,
+    "log_validation_ppl_to_tensorboard": false,
+    "log_world_size_to_tensorboard": false,
+    "logging_level": null,
+    "logit_dtype": null,
+    "logits_load_decode_threads": 4,
+    "logits_load_dir": null,
+    "logits_load_ignore_errors": false,
+    "logits_load_kd_loss_alpha": 1.0,
+    "logits_load_msc_prefetch_depth": 2,
+    "logits_load_prefetch_factor": 3,
+    "logits_save_dir": null,
+    "logits_save_dtype": "fp16",
+    "logits_save_top_k": null,
+    "logits_save_top_p": null,
+    "logits_save_top_p_min_k": 1,
+    "lookahead_allow_objective_change_on_resume": false,
+    "lookahead_attention_backend": "sdpa_auto",
+    "lookahead_cache_layers": null,
+    "lookahead_checkpoint_refiner": false,
+    "lookahead_draft_loss_weight": 0.2,
+    "lookahead_feedback_loss_weight": 1.0,
+    "lookahead_full_attention_interval": 6,
+    "lookahead_future_gradient_mode": "all",
+    "lookahead_future_loss_weight": 0.2,
+    "lookahead_gradient_checkpointing": false,
+    "lookahead_mode": "parallel",
+    "lookahead_normalize_feedback_embedding": false,
+    "lookahead_normalize_feedback_output": false,
+    "lookahead_num_passes": 3,
+    "lookahead_pseudo_initializer": "A2",
+    "lookahead_pseudo_temperature": 1.0,
+    "lookahead_pseudo_top_k": 128,
+    "lookahead_refined_future_loss_weight": 0.0,
+    "lookahead_refiner_ffn_size": 5632,
+    "lookahead_refiner_gate_bias": -2.0,
+    "lookahead_refiner_memory": true,
+    "lookahead_refiner_memory_only": true,
+    "lookahead_refiner_variant": "R0",
+    "lookahead_sliding_window": 2048,
+    "lookahead_z_loss_coefficient": 0.0,
+    "loss_scale": null,
+    "loss_scale_window": 1000,
+    "lr": 0.007470638533717786,
+    "lr_decay_iters": null,
+    "lr_decay_samples": null,
+    "lr_decay_style": "linear",
+    "lr_warmup_fraction": null,
+    "lr_warmup_init": 0.0,
+    "lr_warmup_iters": 0,
+    "lr_warmup_samples": 0,
+    "lr_wsd_decay_iters": 0,
+    "lr_wsd_decay_samples": null,
+    "lr_wsd_decay_style": "linear",
+    "main_grads_dtype": "torch.float32",
+    "main_params_dtype": "torch.float32",
+    "make_vocab_size_divisible_by": 64,
+    "mamba_head_dim": 64,
+    "mamba_inference_conv_states_dtype": "torch.bfloat16",
+    "mamba_inference_ssm_states_dtype": "torch.bfloat16",
+    "mamba_num_groups": 8,
+    "mamba_num_heads": null,
+    "mamba_state_dim": 128,
+    "mamba_training_ssm_states_dtype": null,
+    "manual_gc": false,
+    "manual_gc_eval": true,
+    "manual_gc_interval": 0,
+    "mask_factor": 1.0,
+    "mask_prob": 0.15,
+    "mask_type": "random",
+    "masked_softmax_fusion": true,
+    "max_position_embeddings": 2048,
+    "max_seqlen_per_dp_cp_rank": null,
+    "max_tokens_to_oom": 12000,
+    "megatron_fsdp_enable_fine_grained_param_gather": false,
+    "megatron_fsdp_grad_comm_dtype": null,
+    "megatron_fsdp_main_grads_dtype": null,
+    "megatron_fsdp_main_params_dtype": "torch.float32",
+    "megatron_fsdp_max_pool_double_buffer": false,
+    "megatron_fsdp_version": 1,
+    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
+    "memory_snapshot_path": "snapshot.pickle",
+    "merge_file": null,
+    "metadata_path": null,
+    "mhc_fused_backend": "auto",
+    "mhc_init_gating_factor": 0.01,
+    "mhc_num_residual_streams": 4,
+    "mhc_recompute_layer_num": null,
+    "mhc_sinkhorn_iterations": 20,
+    "micro_batch_size": 8,
+    "microbatch_group_size_per_vp_stage": null,
+    "mid_level_dataset_surplus": 0.005,
+    "min_loss_scale": 1.0,
+    "min_lr": 0.0007470638533717787,
+    "min_offloaded_tensor_size": 1048576,
+    "mla_down_proj_fusion": false,
+    "mlp_chunks_for_prefill": 1,
+    "mlp_chunks_for_training": 1,
+    "mmap_bin_files": true,
+    "mock_data": false,
+    "moe_apply_probs_on_input": false,
+    "moe_aux_loss_coeff": 0.0,
+    "moe_combine_bwd_dtype": "bf16",
+    "moe_deepep_num_sms": null,
+    "moe_dispatch_fwd_dtype": "bf16",
+    "moe_enable_deepep": false,
+    "moe_enable_routing_replay": false,
+    "moe_expert_capacity_factor": null,
+    "moe_expert_rank_capacity_factor": null,
+    "moe_ffn_hidden_size": null,
+    "moe_flex_dispatcher_backend": "deepep",
+    "moe_flex_dispatcher_num_sms": null,
+    "moe_grouped_gemm": false,
+    "moe_hybridep_num_blocks_permute": null,
+    "moe_hybridep_num_blocks_unpermute": null,
+    "moe_hybridep_num_sms": null,
+    "moe_hybridep_num_sms_preprocessing": 108,
+    "moe_hybridep_pad_uneven_dispatch_inputs": false,
+    "moe_input_jitter_eps": null,
+    "moe_latent_size": null,
+    "moe_layer_freq": 1,
+    "moe_layer_recompute": false,
+    "moe_mlp_glu_interleave_size": null,
+    "moe_ncclep_zero_copy": false,
+    "moe_pad_expert_input_to_capacity": false,
+    "moe_pad_experts_for_cuda_graph_inference": false,
+    "moe_paged_stash": false,
+    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
+    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
+    "moe_paged_stash_page_size": 64,
+    "moe_per_layer_logging": false,
+    "moe_permute_fusion": false,
+    "moe_permute_fusion_into_hybridep": false,
+    "moe_router_bias_update_rate": 0.001,
+    "moe_router_dtype": null,
+    "moe_router_enable_expert_bias": false,
+    "moe_router_force_biased": null,
+    "moe_router_force_load_balancing": false,
+    "moe_router_fusion": false,
+    "moe_router_group_topk": null,
+    "moe_router_load_balancing_type": "aux_loss",
+    "moe_router_num_groups": null,
+    "moe_router_padding_for_fp8": false,
+    "moe_router_padding_for_quantization": false,
+    "moe_router_pre_softmax": false,
+    "moe_router_quantile_balancing_ema": 0.0,
+    "moe_router_score_function": "softmax",
+    "moe_router_skip_muon": false,
+    "moe_router_topk": 2,
+    "moe_router_topk_scaling_factor": null,
+    "moe_routing_trace_capture_hidden_states": false,
+    "moe_routing_trace_capture_logits": false,
+    "moe_routing_trace_dump_weights": false,
+    "moe_routing_trace_max_training_iters": null,
+    "moe_routing_trace_path": null,
+    "moe_shared_expert_gate": false,
+    "moe_shared_expert_glu_interleave_size": null,
+    "moe_shared_expert_intermediate_size": null,
+    "moe_shared_expert_overlap": false,
+    "moe_single_grouped_bias": false,
+    "moe_single_grouped_weight": false,
+    "moe_token_dispatcher_type": "allgather",
+    "moe_token_drop_policy": "probs",
+    "moe_upcycling_granularity": 1,
+    "moe_use_grouped_tensor": false,
+    "moe_use_norm_before_up_proj": false,
+    "moe_use_upcycling": false,
+    "moe_z_loss_coeff": null,
+    "mrope_section": null,
+    "mscale": 1.0,
+    "mscale_all_dim": 0.0,
+    "mtp_detach_heads": false,
+    "mtp_hsm": false,
+    "mtp_hybrid_override_pattern": null,
+    "mtp_loss_scaling_factor": 0.1,
+    "mtp_num_layers": null,
+    "mtp_standalone": false,
+    "mtp_use_repeated_layer": false,
+    "multi_latent_attention": false,
+    "multiple_validation_sets": false,
+    "muon_coefficient_type": "quintic",
+    "muon_extra_scale_factor": 1.0,
+    "muon_fp32_matmul_prec": "medium",
+    "muon_momentum": 0.95,
+    "muon_nesterov": true,
+    "muon_num_ns_steps": 5,
+    "muon_scalar_optimizer": "adam",
+    "muon_scale_mode": "spectral",
+    "muon_split_qkv": true,
+    "muon_tp_mode": "duplicated",
+    "muon_use_syrk": false,
+    "mup_attn_scale_power": 1.0,
+    "mup_base_head_dim": null,
+    "mup_base_hidden_size": null,
+    "mup_embedding_mult": 1.0,
+    "mup_output_mult": 1.0,
+    "mup_width_mult": 1.0,
+    "nccl_all_reduce_for_prefill": false,
+    "nccl_communicator_config_path": null,
+    "nccl_ub": false,
+    "no_load_optim": null,
+    "no_load_rng": null,
+    "no_persist_layer_norm": false,
+    "no_rope_freq": null,
+    "no_save_optim": null,
+    "no_save_rng": null,
+    "no_weight_decay_cond_type": null,
+    "non_persistent_ckpt_type": null,
+    "non_persistent_global_ckpt_dir": null,
+    "non_persistent_local_ckpt_algo": "fully_parallel",
+    "non_persistent_local_ckpt_dir": null,
+    "non_persistent_save_interval": null,
+    "normalization": "RMSNorm",
+    "null_tokenizer_eod_id": null,
+    "null_tokenizer_pad_id": -1,
+    "num_attention_heads": 11,
+    "num_channels": 3,
+    "num_classes": 1000,
+    "num_dataset_builder_threads": 1,
+    "num_distributed_optimizer_instances": 1,
+    "num_experts": null,
+    "num_floating_point_operations_so_far": 0,
+    "num_layers": 20,
+    "num_layers_at_end_in_bf16": 1,
+    "num_layers_at_start_in_bf16": 1,
+    "num_layers_per_virtual_pipeline_stage": null,
+    "num_query_groups": 11,
+    "num_speculative_tokens": 0,
+    "num_virtual_stages_per_pipeline_rank": null,
+    "num_workers": 0,
+    "nvtx_ranges": false,
+    "object_storage_cache_path": null,
+    "offload_modules": [],
+    "offset_sampling_seed_by_dp_rank": true,
+    "one_logger_async": false,
+    "one_logger_project": "megatron-lm",
+    "one_logger_run_name": null,
+    "online_data": true,
+    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
+    "online_dataset": "HuggingFaceFW/fineweb-edu",
+    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
+    "online_dataset_subset": "sample-100BT",
+    "online_latency_log_dir": null,
+    "online_pack_sequences": false,
+    "online_prefetch_batches": 16,
+    "online_raw_manifest": null,
+    "online_shuffle_buffer_records": 128,
+    "online_source_mode": "stream",
+    "online_source_split": "train",
+    "online_split_seed": 260421106,
+    "online_text_column": "text",
+    "online_tokenizer_batch_size": 256,
+    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
+    "online_validation_fraction": 0.001,
+    "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json",
+    "onnx_safe": null,
+    "openai_gelu": false,
+    "optimizer": "muonh",
+    "optimizer_cpu_offload": false,
+    "optimizer_cuda_graph": false,
+    "optimizer_offload_fraction": 1.0,
+    "otel_enabled": false,
+    "otel_service_name": null,
+    "otel_span_groups": null,
+    "outer_dp_sharding_strategy": "no_shard",
+    "output_bert_embeddings": false,
+    "overlap_cpu_optimizer_d2h_h2d": false,
+    "overlap_dispatch_backward_with_experts_wgrad": false,
+    "overlap_grad_reduce": false,
+    "overlap_moe_expert_parallel_comm": false,
+    "overlap_p2p_comm": false,
+    "overlap_p2p_comm_warmup_flush": false,
+    "overlap_param_gather": false,
+    "overlap_param_gather_with_optimizer_step": false,
+    "override_ckpt_iteration": null,
+    "override_opt_param_scheduler": false,
+    "pad_vocab_size": true,
+    "padded_vocab_size": 32064,
+    "paper_base_matrix_lr": 0.014,
+    "paper_embedding_lr": 0.3,
+    "paper_norm_lr": 0.005,
+    "paper_other_lr": 0.005,
+    "paper_reference_batch_tokens": 262144,
+    "paper_reference_tokens": 983315200,
+    "paper_reference_width": 640,
+    "paper_transfer": {
+      "batch_tokens": 262144,
+      "reference_batch_tokens": 262144,
+      "reference_tokens": 983315200,
+      "reference_width": 640,
+      "total_tokens": 7000031232,
+      "width": 1408
+    },
+    "paper_unembedding_lr": 0.004,
+    "params_dtype": "torch.bfloat16",
+    "patch_dim": 16,
+    "per_dataset_sequences_path": null,
+    "per_split_data_args_path": null,
+    "perform_initialization": true,
+    "perform_rl_step": false,
+    "phase_transition_iterations": null,
+    "pin_cpu_grads": true,
+    "pin_cpu_params": true,
+    "pipeline_model_parallel_comm_backend": null,
+    "pipeline_model_parallel_layout": null,
+    "pipeline_model_parallel_size": 1,
+    "position_embedding_type": "rope",
+    "pretrained_checkpoint": null,
+    "profile": false,
+    "profile_ranks": [],
+    "profile_step_end": 12,
+    "profile_step_start": 10,
+    "pytorch_profiler_collect_callstack": false,
+    "pytorch_profiler_collect_chakra": false,
+    "pytorch_profiler_collect_shapes": false,
+    "q_lora_rank": null,
+    "qk_clip": false,
+    "qk_clip_alpha": 0.5,
+    "qk_clip_threshold": 100,
+    "qk_head_dim": 128,
+    "qk_l2_norm": false,
+    "qk_layernorm": false,
+    "qk_pos_emb_head_dim": 64,
+    "query_in_block_prob": 0.1,
+    "quick_geglu": false,
+    "rampup_batch_size": null,
+    "rank": 0,
+    "recipe_optimizer": {
+      "betas": [
+        0.8,
+        0.95
+      ],
+      "embedding_lr": 0.3,
+      "eps": 1e-10,
+      "matrix_lr": 0.014,
+      "matrix_weight_decay": 0.0,
+      "max_grad_norm": null,
+      "name": "muonh",
+      "norm_lr": 0.005,
+      "normuon_adjust_lr": "spectral_norm",
+      "normuon_beta2": 0.95,
+      "normuon_momentum": 0.95,
+      "normuon_nesterov": true,
+      "other_lr": 0.005,
+      "scalar_lr": null,
+      "scalar_weight_decay": 0.0,
+      "unembedding_lr": 0.004
+    },
+    "recipe_overrides": {},
+    "recipe_requested_batch_tokens": 262144,
+    "recipe_schedule": {
+      "adamc_weight_decay": false,
+      "cooldown_fraction": 0.0,
+      "cooldown_style": "linear",
+      "final_lr_fraction": 0.1,
+      "kind": "linear",
+      "warmup_steps": 0,
+      "z_loss_coefficient": 0.0
+    },
+    "recompute_granularity": null,
+    "recompute_method": null,
+    "recompute_modules": null,
+    "recompute_num_layers": null,
+    "record_memory_history": false,
+    "record_shapes": false,
+    "refit_execution_batch_bytes": null,
+    "refit_method": "gloo",
+    "relative_attention_max_distance": 128,
+    "relative_attention_num_buckets": 32,
+    "replication": false,
+    "replication_factor": 2,
+    "replication_jump": null,
+    "rerun_mode": "validate_results",
+    "reset_attention_mask": false,
+    "reset_position_ids": false,
+    "result_rejected_tracker_filename": null,
+    "retriever_report_topk_accuracies": [],
+    "retriever_score_scaling": false,
+    "reuse_grad_buf_for_mxfp8_param_ag": false,
+    "rl_consumption_granularity": "B",
+    "rl_default_temperature": 1.0,
+    "rl_default_top_k": -1,
+    "rl_default_top_p": 0,
+    "rl_durable_rollout_bank": false,
+    "rl_generation_lag": null,
+    "rl_importance_sampling_truncation_coef": null,
+    "rl_inference_expert_model_parallel_size": null,
+    "rl_inference_expert_tensor_model_parallel_size": null,
+    "rl_inference_logprobs_is_correction": false,
+    "rl_inference_model_unified_memory_level": 0,
+    "rl_inference_parsers": [],
+    "rl_inference_pipeline_model_parallel_size": null,
+    "rl_inference_tensor_model_parallel_size": null,
+    "rl_kv_cache_management_mode": "persist",
+    "rl_max_inflight_requests": null,
+    "rl_offload_inference_model_weights_when_idle": false,
+    "rl_offload_optimizer_during_inference": false,
+    "rl_partial_rollouts": false,
+    "rl_persist_cuda_graphs": false,
+    "rl_profile": false,
+    "rl_profile_dir": null,
+    "rl_prompts_per_eval": 32,
+    "rl_rollout_bank_dir": null,
+    "rl_rollout_bank_max_bytes": 0,
+    "rl_sequence_packing_algo": "fifo",
+    "rl_sequence_packing_max_sequences_per_bin": 50,
+    "rl_skip_bos_token": false,
+    "rl_submission_granularity": "B",
+    "rl_training_cuda_graphs": false,
+    "rl_use_sequence_packing": false,
+    "rl_verify_model_weights_swap": false,
+    "rope_scaling_factor": 8.0,
+    "rope_type": null,
+    "rotary_base": 10000,
+    "rotary_interleaved": false,
+    "rotary_percent": 1.0,
+    "rotary_scaling_factor": 1.0,
+    "rotary_seq_len_interpolation_factor": null,
+    "run_workload_inspector_server": false,
+    "sample_rate": 1.0,
+    "save": "/root/dl/runtime/script/loopTF-workspace/output/relaunch/20260921-204652_lookahead-memoryonly-relaunch/checkpoints",
+    "save_activations_interval": null,
+    "save_dgrads_interval": null,
+    "save_interval": 4000,
+    "save_params_interval": null,
+    "save_retain_interval": 4000,
+    "save_tokenizer_assets": true,
+    "save_tokens_per_expert_interval": null,
+    "save_wgrads_interval": null,
+    "seed": 42,
+    "seq_length": 2048,
+    "sequence_packing_scheduler": null,
+    "sequence_parallel": false,
+    "sft": false,
+    "sft_mock_dataset_config_json": null,
+    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
+    "sgd_momentum": 0.9,
+    "sharp_enabled_group": null,
+    "short_seq_prob": 0.1,
+    "skip_train": false,
+    "skipped_train_samples": 0,
+    "softmax_type": "vanilla",
+    "spec": null,
+    "special_tokens": null,
+    "split": null,
+    "squared_relu": false,
+    "start_eval_at_iter": null,
+    "start_weight_decay": 0.0,
+    "step_batch_size_schedule": null,
+    "straggler_ctrlr_port": 65535,
+    "straggler_minmax_count": 1,
+    "strict_fsdp_dtensor_load": true,
+    "suggested_communication_unit_size": null,
+    "swiglu": false,
+    "swin_backbone_type": "tiny",
+    "symmetric_ar_type": null,
+    "te_precision_config_file": null,
+    "te_rng_tracker": false,
+    "tensor_model_parallel_size": 1,
+    "tensor_parallel_num_weight_shards": 1,
+    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/relaunch/20260921-204652_lookahead-memoryonly-relaunch/tensorboard",
+    "tensorboard_log_interval": 1,
+    "tensorboard_queue_size": 1000,
+    "test_data_path": null,
+    "test_mode": false,
+    "tiktoken_num_special_tokens": 1000,
+    "tiktoken_pattern": null,
+    "timing_log_level": 0,
+    "timing_log_option": "minmax",
+    "titles_data_path": null,
+    "tokenizer_hf_no_include_special_tokens": false,
+    "tokenizer_hf_no_use_fast": false,
+    "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
+    "tokenizer_sentencepiece_legacy": false,
+    "tokenizer_type": "HuggingFaceTokenizer",
+    "torch_fsdp2_reshard_after_forward": true,
+    "tp_comm_bootstrap_backend": "nccl",
+    "tp_comm_bulk_dgrad": true,
+    "tp_comm_bulk_wgrad": true,
+    "tp_comm_overlap": false,
+    "tp_comm_overlap_ag": true,
+    "tp_comm_overlap_cfg": null,
+    "tp_comm_overlap_rs": true,
+    "tp_comm_overlap_rs_dgrad": false,
+    "tp_comm_split_ag": true,
+    "tp_comm_split_rs": true,
+    "train_data_path": null,
+    "train_iters": 26703,
+    "train_samples": null,
+    "train_sync_interval": null,
+    "training_recipe": "paper2604_21106",
+    "training_recipe_metadata": {
+      "assumptions": [],
+      "backend_kernel_assumptions": {},
+      "base_lrs": {
+        "embedding": 0.3,
+        "matrix": 0.014,
+        "norm": 0.005,
+        "other": 0.005,
+        "unembedding": 0.004
+      },
+      "batch_assumptions": {
+        "global_batch_tokens": 262144,
+        "round_up": false,
+        "sequence_length": 2048
+      },
+      "batch_tokens": 262144,
+      "final_lr_fraction": 0.1,
+      "group_weight_decay": {
+        "embedding": 0.0,
+        "matrix": 0.0,
+        "norm": 0.0,
+        "other": 0.0,
+        "unembedding": 0.0
+      },
+      "lr_transfer": "mup_hyperp_released_source",
+      "model": {
+        "activation": "squared_relu",
+        "activation_checkpointing": false,
+        "allow_objective_change_on_resume": false,
+        "attention": "full_causal",
+        "attention_output_gate": false,
+        "backbone_profile": "paper2604_vanilla",
+        "bias": false,
+        "block_application_multiplier": 6.449999999999999,
+        "cache_layer_indices": [
+          0,
+          10,
+          19
+        ],
+        "checkpoint_refiner": false,
+        "crop_padding_before_loss": true,
+        "draft_loss_weight": 0.2,
+        "dropout": 0.0,
+        "execution_architecture": "lookahead",
+        "feedback_loss_weight": 1.0,
+        "ffn_hidden_size": 5632,
+        "future_gradient_mode": "all",
+        "future_loss_weight": 0.2,
+        "head_dim": 128,
+        "hidden_size": 1408,
+        "initialization": {
+          "embedding_normal_std": 1.0,
+          "head_normal_std": 0.001,
+          "matrix_uniform_bound": "sqrt(3/fan_in)",
+          "norm_scale": 1.0
+        },
+        "logit_softcap": 15.0,
+        "lookahead_mode": "parallel",
+        "model_level_norms": [
+          "embedding",
+          "recur",
+          "final"
+        ],
+        "norm_epsilon": 1e-06,
+        "normalize_feedback_embedding": false,
+        "normalize_feedback_output": false,
+        "num_attention_heads": 11,
+        "num_kv_heads": 11,
+        "num_layers": 20,
+        "objective": "lookahead_parallel_next_token",
+        "padded_vocab_size": 32064,
+        "parameter_counts": {
+          "backbone_total_trainable": 566144128,
+          "feedback_parameters": 3964928,
+          "lookahead_parameters": 105083264,
+          "non_embedding_matrix_formula": 475791360,
+          "non_embedding_paper_N": 475851904,
+          "total_parameters": 671227392,
+          "total_trainable": 671227392
+        },
+        "projection_dtype": "bfloat16 (production)",
+        "pseudo_backend": "sdpa_auto",
+        "pseudo_initializer": "A2",
+        "pseudo_temperature": 1.0,
+        "pseudo_top_k": 128,
+        "qk_norm": "functional_rmsnorm_after_rope",
+        "refined_future_loss_weight": 0.0,
+        "refiner_activation": "silu_gated",
+        "refiner_memory": true,
+        "refiner_memory_only": true,
+        "refiner_variant": "R0",
+        "residual_and_norm_scale_dtype": "float32",
+        "residual_multiplier": 1.0,
+        "rope_base": 10000.0,
+        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+        "tied_embeddings": false,
+        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+        "train_passes": 3,
+        "vocab_size": 32001,
+        "width_scale": 22
+      },
+      "optimizer": {
+        "betas": [
+          0.8,
+          0.95
+        ],
+        "embedding_lr": 0.3,
+        "eps": 1e-10,
+        "matrix_lr": 0.014,
+        "matrix_weight_decay": 0.0,
+        "max_grad_norm": null,
+        "name": "muonh",
+        "norm_lr": 0.005,
+        "normuon_adjust_lr": "spectral_norm",
+        "normuon_beta2": 0.95,
+        "normuon_momentum": 0.95,
+        "normuon_nesterov": true,
+        "other_lr": 0.005,
+        "scalar_lr": null,
+        "scalar_weight_decay": 0.0,
+        "unembedding_lr": 0.004
+      },
+      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+      "paper_settings_overridden": false,
+      "peak_lrs": {
+        "embedding": 0.2022599587389726,
+        "matrix": 0.007470638533717786,
+        "norm": 0.00337099931231621,
+        "other": 0.00337099931231621,
+        "unembedding": 0.0026967994498529684
+      },
+      "recipe": "paper2604_21106",
+      "recipe_overrides": {},
+      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+      "reference_batch_tokens": 262144,
+      "reference_tokens": 983315200,
+      "reference_width": 640,
+      "requested_batch_tokens": 262144,
+      "schedule": {
+        "adamc_weight_decay": false,
+        "cooldown_fraction": 0.0,
+        "cooldown_style": "linear",
+        "final_lr_fraction": 0.1,
+        "kind": "linear",
+        "warmup_steps": 0,
+        "z_loss_coefficient": 0.0
+      },
+      "sequence_length": 2048,
+      "tied_embedding_policy": "unembedding (one AdamW update)",
+      "total_tokens": 7000031232,
+      "warmup_steps": 0,
+      "weight_decay": 0.0,
+      "width": 1408
+    },
+    "transformer_impl": "local",
+    "transformer_pipeline_model_parallel_size": 1,
+    "trust_remote_code": false,
+    "untie_embeddings_and_output_weights": true,
+    "use_checkpoint_args": false,
+    "use_checkpoint_opt_param_scheduler": false,
+    "use_cpu_initialization": null,
+    "use_dist_ckpt": false,
+    "use_dist_ckpt_deprecated": false,
+    "use_distributed_optimizer": false,
+    "use_flash_attn": false,
+    "use_fused_mhc": false,
+    "use_fused_weighted_squared_relu": false,
+    "use_gigatoken": false,
+    "use_gloo_process_groups": true,
+    "use_grouped_gemm_for_dense_mlp": false,
+    "use_grouped_gemm_for_shared_expert": false,
+    "use_kitchen_attention": false,
+    "use_layer_wise_distributed_optimizer": false,
+    "use_layer_wise_param_layout": true,
+    "use_legacy_static_engine": false,
+    "use_mamba_mem_eff_path": true,
+    "use_megatron_fsdp": false,
+    "use_mp_args_from_checkpoint_args": false,
+    "use_mup": false,
+    "use_one_sent_docs": false,
+    "use_persistent_ckpt_worker": false,
+    "use_precision_aware_optimizer": false,
+    "use_pytorch_profiler": false,
+    "use_ring_exchange_p2p": false,
+    "use_rope_scaling": false,
+    "use_rotary_position_embeddings": false,
+    "use_sharp": false,
+    "use_te_activation_func": false,
+    "use_tokenizer_model_from_checkpoint_args": true,
+    "use_torch_fsdp2": false,
+    "use_torch_optimizer_for_cpu_offload": false,
+    "use_tp_pp_dp_mapping": false,
+    "use_transformer_engine_op_fuser": false,
+    "use_varlen_dataset": false,
+    "v_head_dim": 128,
+    "valid_data_path": null,
+    "validation_set_names": null,
+    "variable_seq_lengths": false,
+    "varlen_mock_dataset_config_json": null,
+    "varlen_sbhd_validation": false,
+    "verify_integrity": false,
+    "virtual_pipeline_model_parallel_size": null,
+    "vision_backbone_type": "vit",
+    "vision_pretraining": false,
+    "vision_pretraining_type": "classify",
+    "vocab_extra_ids": 0,
+    "vocab_file": null,
+    "vocab_size": 32001,
+    "wandb_entity": "LoopTF-4-CSPs",
+    "wandb_exp_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b",
+    "wandb_project": "loopTF-workspace",
+    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/relaunch/20260921-204652_lookahead-memoryonly-relaunch/wandb",
+    "weight_decay": 0.0,
+    "weight_decay_incr_style": "constant",
+    "wgrad_deferral_limit": 0,
+    "window_attn_skip_freq": null,
+    "window_size": null,
+    "world_size": 16,
+    "yaml_cfg": null,
+    "yarn_beta_fast": null,
+    "yarn_beta_slow": null,
+    "yarn_correction_range_round_to_int": null,
+    "yarn_original_max_position_embeddings": null
+  },
+  "model_config": {
+    "attention_dropout": 0.0,
+    "attn_implementation": "sdpa",
+    "backbone_profile": "paper2604_vanilla",
+    "gradient_checkpointing": false,
+    "hf_config_overrides": {},
+    "hidden_size": 1408,
+    "initializer_range": 0.02,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 2048,
+    "model_type": "full_bandwidth",
+    "num_attention_heads": 11,
+    "num_hidden_layers": 20,
+    "num_key_value_heads": 11,
+    "padded_vocab_size": 32064,
+    "paper_full_attention_interval": 6,
+    "paper_sliding_window": 2048,
+    "pretrained_model_name_or_path": null,
+    "revision": null,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000,
+    "tie_word_embeddings": false,
+    "trust_remote_code": false,
+    "vocab_size": 32001
+  },
+  "native_reconstruction": {
+    "active_vocab_size": 32001,
+    "architecture": "lookahead",
+    "attention_backend": "native PyTorch SDPA; Look-ahead backend=sdpa",
+    "ignored_extra_state": [],
+    "max_supported_length": 20480,
+    "model_config": {
+      "attention_dropout": 0.0,
+      "attn_implementation": "sdpa",
+      "backbone_profile": "paper2604_vanilla",
+      "gradient_checkpointing": false,
+      "hf_config_overrides": {},
+      "hidden_size": 1408,
+      "initializer_range": 0.02,
+      "intermediate_size": 5632,
+      "max_position_embeddings": 2048,
+      "model_type": "full_bandwidth",
+      "num_attention_heads": 11,
+      "num_hidden_layers": 20,
+      "num_key_value_heads": 11,
+      "padded_vocab_size": 32064,
+      "paper_full_attention_interval": 6,
+      "paper_sliding_window": 2048,
+      "pretrained_model_name_or_path": null,
+      "revision": null,
+      "rms_norm_eps": 1e-06,
+      "rope_theta": 10000,
+      "tie_word_embeddings": false,
+      "trust_remote_code": false,
+      "vocab_size": 32001
+    },
+    "policy": {
+      "allow_objective_change_on_resume": false,
+      "attention_backend": "sdpa",
+      "cache_layer_indices": null,
+      "checkpoint_refiner": false,
+      "draft_loss_weight": 0.2,
+      "feedback_loss_weight": 1.0,
+      "future_gradient_mode": "all",
+      "future_loss_weight": 0.2,
+      "mode": "parallel",
+      "normalize_feedback_embedding": false,
+      "normalize_feedback_output": false,
+      "num_passes": 3,
+      "pseudo_initializer": "A2",
+      "pseudo_temperature": 1.0,
+      "pseudo_top_k": 128,
+      "refined_future_loss_weight": 0.0,
+      "refiner_gate_bias": -2.0,
+      "refiner_intermediate_size": 5632,
+      "refiner_memory": true,
+      "refiner_memory_only": true,
+      "refiner_variant": "R0"
+    },
+    "prediction": "fixed final-pass full-sequence logits",
+    "step": 26703
+  },
+  "original_training_run_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b",
+  "parameter_counts": {
+    "backbone_total_trainable": 566144128,
+    "feedback_parameters": 3964928,
+    "lookahead_parameters": 105083264,
+    "non_embedding_matrix_formula": 475791360,
+    "non_embedding_paper_N": 475851904,
+    "total_parameters": 671227392,
+    "total_trainable": 671227392
+  },
+  "pipeline_model_parallel_size": 1,
+  "run_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci",
+  "source_run_directory": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/output/relaunch/20260921-204652_lookahead-memoryonly-relaunch",
+  "tensor_model_parallel_size": 1,
+  "tokenizer": {
+    "added_tokens": [
+      "<|assistant_end|>"
+    ],
+    "repository": "NousResearch/Llama-2-7b-hf",
+    "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
+    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d"
+  },
+  "training_budget_tokens": 7000031232,
+  "training_code_revision": "7500042108313496c8e252eb2849f87f934c7b8f",
+  "training_tokens": 7000031232,
+  "wandb": {
+    "entity": "LoopTF-4-CSPs",
+    "project": "loopTF-workspace",
+    "run_id": "332i1pci",
+    "run_name": "Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b"
+  }
+}

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/launch_config.env

@@ -0,0 +1,5 @@
+# Same Look-ahead recipe; replace the real-bank read by ungated A_new.
+source "$(dirname "${BASH_SOURCE[0]}")/Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemory-7b.env"
+LOOKAHEAD_REFINER_MEMORY_ONLY=true
+RUN_NAME=Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b
+WANDB_TAGS=product,lookahead,paper2604,476m,7b,k3,la2,d0p2,p0p2,f0,a2,r0,refiner_memory_only

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/manifest.json

@@ -0,0 +1,25 @@
+{
+  "files": {
+    "README.md": {
+      "sha256": "4505052ae9b75f0d00d10285906eb3a79bea9808ec965bd0753d11dd00f0fc45",
+      "size": 581
+    },
+    "config.json": {
+      "sha256": "430687d6edb05e8ed328fa6c4093269d7fb3a6e18b02d3de948815620817ef37",
+      "size": 50796
+    },
+    "launch_config.env": {
+      "sha256": "b4cb96d810799641fea03e03bd544efe3d3ce44290919bc2070f5082325d4cf9",
+      "size": 376
+    },
+    "model.safetensors": {
+      "sha256": "46f96f614661eab64d84c2a4be97873fae8e7c29b2c7791a07cd91c5266ed58f",
+      "size": 1342621344
+    },
+    "training_recipe.json": {
+      "sha256": "deb8b17953a0e3fd07712bf810846a5204df37fdb9f7936d5405c309e42d37d3",
+      "size": 4536
+    }
+  },
+  "note": "manifest.json does not hash itself"
+}

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46f96f614661eab64d84c2a4be97873fae8e7c29b2c7791a07cd91c5266ed58f
+size 1342621344

Lookahead-K3-LA2-A2-R0-D0p2-P0p2-F0-seed42-refinerMemoryOnly-7b--332i1pci/step_26703/training_recipe.json

@@ -0,0 +1,155 @@
+{
+  "recipe": "paper2604_21106",
+  "width": 1408,
+  "total_tokens": 7000031232,
+  "batch_tokens": 262144,
+  "reference_width": 640,
+  "reference_tokens": 983315200,
+  "reference_batch_tokens": 262144,
+  "base_lrs": {
+    "matrix": 0.014,
+    "embedding": 0.3,
+    "unembedding": 0.004,
+    "norm": 0.005,
+    "other": 0.005
+  },
+  "peak_lrs": {
+    "matrix": 0.007470638533717786,
+    "embedding": 0.2022599587389726,
+    "unembedding": 0.0026967994498529684,
+    "norm": 0.00337099931231621,
+    "other": 0.00337099931231621
+  },
+  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
+  "tied_embedding_policy": "unembedding (one AdamW update)",
+  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
+  "weight_decay": 0.0,
+  "warmup_steps": 0,
+  "final_lr_fraction": 0.1,
+  "optimizer": {
+    "name": "muonh",
+    "matrix_lr": 0.014,
+    "scalar_lr": null,
+    "matrix_weight_decay": 0.0,
+    "scalar_weight_decay": 0.0,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.004,
+    "norm_lr": 0.005,
+    "other_lr": 0.005,
+    "betas": [
+      0.8,
+      0.95
+    ],
+    "eps": 1e-10,
+    "normuon_momentum": 0.95,
+    "normuon_beta2": 0.95,
+    "normuon_nesterov": true,
+    "normuon_adjust_lr": "spectral_norm",
+    "max_grad_norm": null
+  },
+  "schedule": {
+    "kind": "linear",
+    "warmup_steps": 0,
+    "cooldown_fraction": 0.0,
+    "cooldown_style": "linear",
+    "final_lr_fraction": 0.1,
+    "z_loss_coefficient": 0.0,
+    "adamc_weight_decay": false
+  },
+  "group_weight_decay": {
+    "matrix": 0.0,
+    "embedding": 0.0,
+    "unembedding": 0.0,
+    "norm": 0.0,
+    "other": 0.0
+  },
+  "recipe_overrides": {},
+  "requested_batch_tokens": 262144,
+  "sequence_length": 2048,
+  "batch_assumptions": {
+    "global_batch_tokens": 262144,
+    "sequence_length": 2048,
+    "round_up": false
+  },
+  "lr_transfer": "mup_hyperp_released_source",
+  "assumptions": [],
+  "backend_kernel_assumptions": {},
+  "paper_settings_overridden": false,
+  "model": {
+    "backbone_profile": "paper2604_vanilla",
+    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
+    "width_scale": 22,
+    "hidden_size": 1408,
+    "num_layers": 20,
+    "head_dim": 128,
+    "num_attention_heads": 11,
+    "num_kv_heads": 11,
+    "ffn_hidden_size": 5632,
+    "activation": "squared_relu",
+    "attention": "full_causal",
+    "attention_output_gate": false,
+    "qk_norm": "functional_rmsnorm_after_rope",
+    "norm_epsilon": 1e-06,
+    "model_level_norms": [
+      "embedding",
+      "recur",
+      "final"
+    ],
+    "rope_base": 10000.0,
+    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
+    "bias": false,
+    "dropout": 0.0,
+    "residual_multiplier": 1.0,
+    "residual_and_norm_scale_dtype": "float32",
+    "projection_dtype": "bfloat16 (production)",
+    "tied_embeddings": false,
+    "vocab_size": 32001,
+    "padded_vocab_size": 32064,
+    "logit_softcap": 15.0,
+    "crop_padding_before_loss": true,
+    "initialization": {
+      "embedding_normal_std": 1.0,
+      "head_normal_std": 0.001,
+      "matrix_uniform_bound": "sqrt(3/fan_in)",
+      "norm_scale": 1.0
+    },
+    "parameter_counts": {
+      "non_embedding_matrix_formula": 475791360,
+      "non_embedding_paper_N": 475851904,
+      "total_trainable": 671227392,
+      "feedback_parameters": 3964928,
+      "backbone_total_trainable": 566144128,
+      "lookahead_parameters": 105083264,
+      "total_parameters": 671227392
+    },
+    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
+    "execution_architecture": "lookahead",
+    "objective": "lookahead_parallel_next_token",
+    "lookahead_mode": "parallel",
+    "train_passes": 3,
+    "feedback_loss_weight": 1.0,
+    "cache_layer_indices": [
+      0,
+      10,
+      19
+    ],
+    "refiner_activation": "silu_gated",
+    "pseudo_backend": "sdpa_auto",
+    "activation_checkpointing": false,
+    "checkpoint_refiner": false,
+    "block_application_multiplier": 6.449999999999999,
+    "future_loss_weight": 0.2,
+    "future_gradient_mode": "all",
+    "draft_loss_weight": 0.2,
+    "refined_future_loss_weight": 0.0,
+    "pseudo_initializer": "A2",
+    "pseudo_top_k": 128,
+    "pseudo_temperature": 1.0,
+    "normalize_feedback_embedding": false,
+    "normalize_feedback_output": false,
+    "refiner_variant": "R0",
+    "refiner_memory": true,
+    "refiner_memory_only": true,
+    "allow_objective_change_on_resume": false
+  }
+}