Remove verified legacy Loop paths after rename

loop_ifm_freshkv_7b_20260920-213933/step_26703/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_freshkv_7b_20260920-213933 / step_26703
-
-Native fresh-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Final validation metrics are recorded from this run in config.json.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 recomputes fresh KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_freshkv_7b_20260920-213933/step_26703/config.json

@@ -1,1431 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_freshkv_7b_20260920-213933",
-  "iteration": 26703,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_freshkv_7b_20260920-213933",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": false,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 3417984,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 68608,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": false,
-        "depth_kv_policy": "recompute_each_loop",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": false,
-      "depth_kv_policy": "recompute_each_loop",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 26702
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 7000031232,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": false,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_freshkv_7b_20260920-213933",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README.",
-  "final_validation": {
-    "eval/final_val/pass_1_ce": 2.293280601501465,
-    "eval/final_val/pass_1_ppl": 9.907386613462691,
-    "eval/final_val/pass_2_ce": 2.2767744064331055,
-    "eval/final_val/pass_2_ppl": 9.74519561796341,
-    "eval/final_val/pass_3_ce": 2.276322364807129,
-    "eval/final_val/pass_3_ppl": 9.74079137941538,
-    "eval/final_val/final_ce": 2.276322364807129,
-    "eval/final_val/final_ppl": 9.74079137941538,
-    "eval/final_val/pass_count": 3,
-    "eval/checkpoint_step": 26703,
-    "eval/is_final": true,
-    "eval/final_val/num_sequences": 34560,
-    "eval/final_val/num_tokens": 70778880,
-    "eval/final_val/num_input_tokens": 70778880,
-    "eval/final_val/dataset_id": "config-cb8c1cc82eb53c5c"
-  }
-}

loop_ifm_freshkv_7b_20260920-213933/step_26703/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=false
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_freshkv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,freshkv

loop_ifm_freshkv_7b_20260920-213933/step_26703/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3416,
-      "sha256": "4dfeba7349838f5bba2b8948625553c3e3dade8bc5a1a6d3f38de467ae752ab1"
-    },
-    "config.json": {
-      "size": 54867,
-      "sha256": "f44859fba61361b4fd88175d182ec90e92cf3cdaa34ccc188168724d1bb16185"
-    },
-    "launch_config.env": {
-      "size": 534,
-      "sha256": "a5b0afb57b1d34b2c1cee722ff79fc6eeb4b7b246493f29fb0414397be23e7bd"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "5e819f113b4014446733cb8bda9b00bbb550aa25cd5b97a995f40a165d2d3286"
-    },
-    "training_recipe.json": {
-      "size": 5092,
-      "sha256": "38b5c1786244228c996724ea6ee630898f5bcb898774abc0e3aadff1326dafa9"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_freshkv_7b_20260920-213933/step_26703/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5e819f113b4014446733cb8bda9b00bbb550aa25cd5b97a995f40a165d2d3286
-size 1310894448

loop_ifm_freshkv_7b_20260920-213933/step_26703/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": false,
-    "depth_kv_policy": "recompute_each_loop",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_12000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_12000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_12000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 12000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 1536000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 30976,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 11999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 3145728000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_12000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_12000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3451,
-      "sha256": "973a0d9814fec90cc99260e9ff343ca96b36b0c54475798024856f4f61ca0cb6"
-    },
-    "config.json": {
-      "size": 54156,
-      "sha256": "c21201abf80edb8a3cf18ffed6c290c2e17d829441913909c4d112128682b38f"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "d153a3f096756906756fdb7fbb5b0f491b7d9cf2ec448876f0d2a6b1e386f811"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_12000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d153a3f096756906756fdb7fbb5b0f491b7d9cf2ec448876f0d2a6b1e386f811
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_12000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_16000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_16000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 16000, consumed 4,194,304,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_16000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 16000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 2048000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 41216,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 15999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 4194304000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_16000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_16000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3451,
-      "sha256": "037e15d0f02ba5fa5205be337a7c0207e604e4f540bd0e320229849b16bfe9e2"
-    },
-    "config.json": {
-      "size": 54156,
-      "sha256": "7e47b2b81c764d67cb70d8f06c2ec10dc909421c641aedeb06509b7a6ecf3752"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "7b6362546d56c93fc05ad7c5285b7a90e0b203a7dc022906efc694a3b703b7b5"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_16000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7b6362546d56c93fc05ad7c5285b7a90e0b203a7dc022906efc694a3b703b7b5
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_16000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_20000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_20000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 20000, consumed 5,242,880,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_20000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 20000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 2560000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 51456,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 19999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 5242880000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_20000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_20000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3451,
-      "sha256": "10e6715718b11885ba343029d01e3f3d562da7e03c71eebcae8118e2728d80c0"
-    },
-    "config.json": {
-      "size": 54156,
-      "sha256": "9ed6d74bdbb56315a74a534022f6bc02c65c519c19c59fb50e622c2ebce344cf"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "35fede4757419e4449384de7e64a0f4a8e92305937fdf6dde9cc00c3cbf6b175"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_20000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:35fede4757419e4449384de7e64a0f4a8e92305937fdf6dde9cc00c3cbf6b175
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_20000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_24000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_24000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 24000, consumed 6,291,456,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_24000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 24000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 3072000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 61696,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 23999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 6291456000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_24000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_24000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3451,
-      "sha256": "027f6622ec6f3d649eab9ab5e6f696bc166fc7b354bb08676f2c984f8c1ab483"
-    },
-    "config.json": {
-      "size": 54156,
-      "sha256": "2d92ea08687ce5bff6bec73c14a3562c062769c291d1d4031380adc88c3be103"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "dad05a5a544259a3e41d069b609790a4d38d3c1c955c89eb8b40cb3a40433f01"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_24000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dad05a5a544259a3e41d069b609790a4d38d3c1c955c89eb8b40cb3a40433f01
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_24000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_26703/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_26703
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_26703/config.json

@@ -1,1431 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 26703,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 3417984,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 68608,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 26702
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 7000031232,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README.",
-  "final_validation": {
-    "eval/final_val/pass_1_ce": 2.304596424102783,
-    "eval/final_val/pass_1_ppl": 10.020133551919583,
-    "eval/final_val/pass_2_ce": 2.280852794647217,
-    "eval/final_val/pass_2_ppl": 9.785021466348496,
-    "eval/final_val/pass_3_ce": 2.279691696166992,
-    "eval/final_val/pass_3_ppl": 9.773666686079672,
-    "eval/final_val/final_ce": 2.279691696166992,
-    "eval/final_val/final_ppl": 9.773666686079672,
-    "eval/final_val/pass_count": 3,
-    "eval/checkpoint_step": 26703,
-    "eval/is_final": true,
-    "eval/final_val/num_sequences": 34560,
-    "eval/final_val/num_tokens": 70778880,
-    "eval/final_val/num_input_tokens": 70778880,
-    "eval/final_val/dataset_id": "config-cb8c1cc82eb53c5c"
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_26703/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_26703/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3451,
-      "sha256": "a2d145dd6896a99b66a0d8e602b2c4b0f3e2caf72f2d528cffbc1fe5603ff7fb"
-    },
-    "config.json": {
-      "size": 54888,
-      "sha256": "7c0746b2f1cca12d677f6dbec74634baa42401883d2890bd963077a3ff4059b0"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "8d961dacfd1cf67f11536a4eba6c73df8e822788b840fd99ed10d73330b5d976"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_26703/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8d961dacfd1cf67f11536a4eba6c73df8e822788b840fd99ed10d73330b5d976
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_26703/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_4000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_4000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 4000, consumed 1,048,576,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_4000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 4000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 512000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 10496,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 3999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 1048576000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_4000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_4000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3449,
-      "sha256": "4eb470c9b5019a8ae3131b705bde16633ccf14ccc1e5fb9434c7c49c8fe4d57e"
-    },
-    "config.json": {
-      "size": 54153,
-      "sha256": "f60eda6d58a8aae9a0eab9058cb334d821090fbb3bdc3c01a46537543070affc"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "f0c51d09d782f0b8b10451a6f6f85de49ca4baa7ae0777f84de5955d0ba01d0a"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_4000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f0c51d09d782f0b8b10451a6f6f85de49ca4baa7ae0777f84de5955d0ba01d0a
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_4000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

loop_ifm_sharekv_7b_20260920-153012/step_8000/README.md

@@ -1,50 +0,0 @@
-# loop_ifm_sharekv_7b_20260920-153012 / step_8000
-
-Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 8000, consumed 2,097,152,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

loop_ifm_sharekv_7b_20260920-153012/step_8000/config.json

@@ -1,1414 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "loop_ifm_sharekv_7b_20260920-153012",
-  "iteration": 8000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1",
-  "source_provenance": "Training used training_code_revision with an empty saved code.diff.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 655369088,
-    "trainable": 655369088
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 8,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints",
-    "save_interval": 4000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": 4000,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 8,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_layers": null,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "depth_share_kv": true,
-    "loop_input_write": "ifm_diagonal",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": false,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 1024000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 20736,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 655369088,
-          "feedback_parameters": 3967744,
-          "loop_input_write_parameters": 1985280,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "depth_share_kv": true,
-        "depth_kv_policy": "first_loop_per_layer_connected",
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_input_write": "ifm_diagonal",
-        "loop_input_write_source": "raw_token_embeddings",
-        "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-        "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-        "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": false,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 655369088,
-        "feedback_parameters": 3967744,
-        "loop_input_write_parameters": 1985280,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "depth_share_kv": true,
-      "depth_kv_policy": "first_loop_per_layer_connected",
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_input_write": "ifm_diagonal",
-      "loop_input_write_source": "raw_token_embeddings",
-      "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-      "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-      "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": false,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 7999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 2097152000,
-  "training_budget_tokens": 7000031232,
-  "ifm_diagonal": {
-    "source": "raw_token_embeddings",
-    "placement": "before_both_depth_loops",
-    "shared_projection_and_gating": true
-  },
-  "depth_share_kv": true,
-  "effective_launch_overrides": {
-    "NNODES": 2,
-    "MICRO_BATCH_SIZE": 8,
-    "GLOBAL_BATCH_SIZE": 128,
-    "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012",
-    "SAVE_INTERVAL": 4000,
-    "SAVE_RETAIN_INTERVAL": 4000,
-    "ENABLE_RECOMPUTE": false,
-    "LOOP_ADAPTER_CHECKPOINT_REFINER": false
-  },
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}

loop_ifm_sharekv_7b_20260920-153012/step_8000/launch_config.env

@@ -1,12 +0,0 @@
-# IFM raw-token input injection before every depth loop; fresh 16-GPU arm.
-source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env"
-LOOP_INPUT_WRITE=ifm_diagonal
-DEPTH_SHARE_KV=true
-NNODES=2
-MICRO_BATCH_SIZE=8
-ENABLE_RECOMPUTE=false
-LOOP_ADAPTER_CHECKPOINT_REFINER=false
-SAVE_RETAIN_INTERVAL=4000
-RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv
-WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3
-WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv

loop_ifm_sharekv_7b_20260920-153012/step_8000/manifest.json

@@ -1,25 +0,0 @@
-{
-  "files": {
-    "README.md": {
-      "size": 3449,
-      "sha256": "76a7963c72a7dfb46270f7e2d4c8b1e730ef346062d0624d5aaad3a9e5b7c8d1"
-    },
-    "config.json": {
-      "size": 54154,
-      "sha256": "358cd038eea825c018625fbaace647c0b7843987cd9e8bb77f235629c06cf644"
-    },
-    "launch_config.env": {
-      "size": 533,
-      "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72"
-    },
-    "model.safetensors": {
-      "size": 1310894448,
-      "sha256": "c3674b08899fb3e3b7c10f1366f998f5ce7f148df52f05060f1fc5743b2972ab"
-    },
-    "training_recipe.json": {
-      "size": 5102,
-      "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d"
-    }
-  },
-  "note": "Manifest does not hash itself."
-}

loop_ifm_sharekv_7b_20260920-153012/step_8000/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c3674b08899fb3e3b7c10f1366f998f5ce7f148df52f05060f1fc5743b2972ab
-size 1310894448

loop_ifm_sharekv_7b_20260920-153012/step_8000/training_recipe.json

@@ -1,163 +0,0 @@
-{
-  "recipe": "paper2604_21106",
-  "width": 1408,
-  "total_tokens": 7000031232,
-  "batch_tokens": 262144,
-  "reference_width": 640,
-  "reference_tokens": 983315200,
-  "reference_batch_tokens": 262144,
-  "base_lrs": {
-    "matrix": 0.014,
-    "embedding": 0.3,
-    "unembedding": 0.004,
-    "norm": 0.005,
-    "other": 0.005
-  },
-  "peak_lrs": {
-    "matrix": 0.007470638533717786,
-    "embedding": 0.2022599587389726,
-    "unembedding": 0.0026967994498529684,
-    "norm": 0.00337099931231621,
-    "other": 0.00337099931231621
-  },
-  "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-  "tied_embedding_policy": "unembedding (one AdamW update)",
-  "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-  "weight_decay": 0.0,
-  "warmup_steps": 0,
-  "final_lr_fraction": 0.1,
-  "optimizer": {
-    "name": "muonh",
-    "matrix_lr": 0.014,
-    "scalar_lr": null,
-    "matrix_weight_decay": 0.0,
-    "scalar_weight_decay": 0.0,
-    "embedding_lr": 0.3,
-    "unembedding_lr": 0.004,
-    "norm_lr": 0.005,
-    "other_lr": 0.005,
-    "betas": [
-      0.8,
-      0.95
-    ],
-    "eps": 1e-10,
-    "normuon_momentum": 0.95,
-    "normuon_beta2": 0.95,
-    "normuon_nesterov": true,
-    "normuon_adjust_lr": "spectral_norm",
-    "max_grad_norm": null
-  },
-  "schedule": {
-    "kind": "linear",
-    "warmup_steps": 0,
-    "cooldown_fraction": 0.0,
-    "cooldown_style": "linear",
-    "final_lr_fraction": 0.1,
-    "z_loss_coefficient": 0.0,
-    "adamc_weight_decay": false
-  },
-  "group_weight_decay": {
-    "matrix": 0.0,
-    "embedding": 0.0,
-    "unembedding": 0.0,
-    "norm": 0.0,
-    "other": 0.0
-  },
-  "recipe_overrides": {},
-  "requested_batch_tokens": 262144,
-  "sequence_length": 2048,
-  "batch_assumptions": {
-    "global_batch_tokens": 262144,
-    "sequence_length": 2048,
-    "round_up": false
-  },
-  "lr_transfer": "mup_hyperp_released_source",
-  "assumptions": [],
-  "backend_kernel_assumptions": {},
-  "paper_settings_overridden": false,
-  "model": {
-    "backbone_profile": "paper2604_vanilla",
-    "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-    "width_scale": 22,
-    "hidden_size": 1408,
-    "num_layers": 20,
-    "head_dim": 128,
-    "num_attention_heads": 11,
-    "num_kv_heads": 11,
-    "ffn_hidden_size": 5632,
-    "activation": "squared_relu",
-    "attention": "full_causal",
-    "attention_output_gate": false,
-    "qk_norm": "functional_rmsnorm_after_rope",
-    "norm_epsilon": 1e-06,
-    "model_level_norms": [
-      "embedding",
-      "recur",
-      "final"
-    ],
-    "rope_base": 10000.0,
-    "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-    "bias": false,
-    "dropout": 0.0,
-    "residual_multiplier": 1.0,
-    "residual_and_norm_scale_dtype": "float32",
-    "projection_dtype": "bfloat16 (production)",
-    "tied_embeddings": false,
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "logit_softcap": 15.0,
-    "crop_padding_before_loss": true,
-    "initialization": {
-      "embedding_normal_std": 1.0,
-      "head_normal_std": 0.001,
-      "matrix_uniform_bound": "sqrt(3/fan_in)",
-      "norm_scale": 1.0
-    },
-    "parameter_counts": {
-      "non_embedding_matrix_formula": 475791360,
-      "non_embedding_paper_N": 475851904,
-      "total_trainable": 655369088,
-      "feedback_parameters": 3967744,
-      "loop_input_write_parameters": 1985280,
-      "loop_adapter_parameters": 83271936
-    },
-    "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-    "execution_architecture": "full_bandwidth_loop",
-    "objective": "latent_feedback_causal_lm",
-    "feedback_enabled": true,
-    "train_passes": 3,
-    "eval_passes": 3,
-    "feedback_pass_mixture": {
-      "3": 1.0
-    },
-    "feedback_progressive_stages": [],
-    "depth_num_loops": 2,
-    "logical_layers_per_pass": 40,
-    "shared_layer_weights": true,
-    "depth_share_kv": true,
-    "depth_kv_policy": "first_loop_per_layer_connected",
-    "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-    "loop_input_write": "ifm_diagonal",
-    "loop_input_write_source": "raw_token_embeddings",
-    "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb",
-    "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes",
-    "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)",
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter": {
-      "ffn_size": 5632,
-      "state_input": "second-loop hidden states after norm_recur and norm_final",
-      "terminal_norms": true,
-      "attention_backend": "sdpa_auto",
-      "checkpoint_refiner": false,
-      "bank_layers": [
-        0,
-        10,
-        19
-      ],
-      "bank_source": "first depth loop, fresh on each feedback pass",
-      "weight_sharing": "one adapter shared across feedback passes"
-    },
-    "block_application_multiplier_per_pass": 2.15,
-    "block_application_multiplier": 6.449999999999999
-  }
-}

paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/README.md

@@ -1,50 +0,0 @@
-# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_12000
-
-Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here.
-
-Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132.
-
-Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners.
-
-## Native CPU loading
-
-```python
-import json, torch, importlib, tempfile
-from argparse import Namespace
-from megatron.core import parallel_state
-from megatron.core.process_groups_config import ProcessGroupCollection
-from full_bandwidth_loop_tf.megatron.config import build_model_config
-from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder
-from safetensors.torch import load_file
-
-def decode(x):
-    if isinstance(x, dict):
-        if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"])
-        if "__dataclass__" in x:
-            module, cls, values = x["__dataclass__"]
-            return getattr(importlib.import_module(module), cls)(**decode(values))
-        if "__enum__" in x:
-            module, cls, member = x["__enum__"]
-            return getattr(importlib.import_module(module), cls)[member]
-        return {k: decode(v) for k, v in x.items()}
-    if isinstance(x, list): return [decode(v) for v in x]
-    return x
-
-c = json.load(open("config.json"))
-a = Namespace(**decode(c["model_arguments"]))
-a.use_cpu_initialization = True
-a.perform_initialization = False
-cfg = build_model_config(a)
-# Standalone CPU loading; use your existing groups in an initialized runtime.
-with tempfile.TemporaryDirectory() as tmp:
-    torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1)
-    parallel_state.initialize_model_parallel()
-    with torch.device("meta"):
-        model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups())
-    state = load_file("model.safetensors") | c["non_tensor_state"]
-    model.load_state_dict(state, strict=True, assign=True)
-    model.eval()
-    torch.distributed.destroy_process_group()
-```
-
-All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself.

paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/config.json

@@ -1,1379 +0,0 @@
-{
-  "checkpoint_format": "safetensors_native_megatron_state_dict",
-  "architecture": "full_bandwidth_loop",
-  "backbone_profile": "paper2604_vanilla",
-  "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm",
-  "iteration": 12000,
-  "checkpoint_version": 3.0,
-  "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83",
-  "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941",
-  "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.",
-  "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713",
-  "parameter_counts": {
-    "total": 653383808,
-    "trainable": 653383808
-  },
-  "model_arguments": {
-    "tensor_model_parallel_size": 1,
-    "tensor_parallel_num_weight_shards": 1,
-    "pipeline_model_parallel_comm_backend": null,
-    "pipeline_model_parallel_size": 1,
-    "sequence_parallel": false,
-    "context_parallel_size": 1,
-    "hierarchical_context_parallel_sizes": null,
-    "expert_model_parallel_size": 1,
-    "expert_tensor_parallel_size": 1,
-    "expert_tensor_parallel_num_weight_shards": 1,
-    "perform_initialization": true,
-    "use_cpu_initialization": null,
-    "fp16": false,
-    "bf16": true,
-    "deterministic_mode": false,
-    "tp_comm_overlap": false,
-    "tp_comm_bulk_wgrad": true,
-    "tp_comm_bulk_dgrad": true,
-    "tp_comm_overlap_ag": true,
-    "tp_comm_overlap_rs": true,
-    "tp_comm_overlap_rs_dgrad": false,
-    "tp_comm_split_ag": true,
-    "tp_comm_split_rs": true,
-    "cross_entropy_loss_fusion": false,
-    "cross_entropy_fusion_impl": "native",
-    "tp_comm_bootstrap_backend": "nccl",
-    "overlap_moe_expert_parallel_comm": false,
-    "delay_wgrad_compute": false,
-    "overlap_dispatch_backward_with_experts_wgrad": false,
-    "ep_overlap_early_attn_memory_release": false,
-    "use_ring_exchange_p2p": false,
-    "defer_embedding_wgrad_compute": false,
-    "wgrad_deferral_limit": 0,
-    "overlap_p2p_comm_warmup_flush": false,
-    "microbatch_group_size_per_vp_stage": null,
-    "mtp_standalone": false,
-    "cpu_offloading_num_layers": 0,
-    "cpu_offloading_retain_pinned_cpu_buffers": false,
-    "num_layers": 20,
-    "mtp_num_layers": null,
-    "mtp_loss_scaling_factor": 0.1,
-    "mtp_use_repeated_layer": false,
-    "mtp_detach_heads": false,
-    "mtp_hsm": false,
-    "mtp_hybrid_override_pattern": null,
-    "account_for_embedding_in_pipeline_split": false,
-    "account_for_loss_in_pipeline_split": false,
-    "hidden_size": 1408,
-    "num_attention_heads": 11,
-    "attention_backend": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "AttnBackend",
-        "auto"
-      ]
-    },
-    "flash_attention_version": null,
-    "softmax_type": "vanilla",
-    "num_query_groups": 11,
-    "ffn_hidden_size": 5632,
-    "kv_channels": 128,
-    "hidden_dropout": 0.0,
-    "attention_dropout": 0.0,
-    "fp32_residual_connection": false,
-    "apply_residual_connection_post_layernorm": false,
-    "layernorm_epsilon": 1e-06,
-    "layernorm_zero_centered_gamma": false,
-    "add_bias_linear": false,
-    "add_qkv_bias": false,
-    "glu_linear_offset": 0.0,
-    "activation_func_clamp_value": null,
-    "rotary_interleaved": false,
-    "normalization": "RMSNorm",
-    "qk_layernorm": false,
-    "qk_l2_norm": false,
-    "qk_clip": false,
-    "qk_clip_alpha": 0.5,
-    "qk_clip_threshold": 100,
-    "attention_output_gate": false,
-    "calculate_per_token_loss": false,
-    "multi_latent_attention": false,
-    "experimental_attention_variant": null,
-    "dsa_indexer_n_heads": null,
-    "dsa_indexer_head_dim": null,
-    "dsa_indexer_topk": null,
-    "dsa_indexer_topk_freq": 1,
-    "dsa_indexer_skip_topk_offset": 0,
-    "dsa_indexer_loss_coeff": null,
-    "dsa_indexer_use_sparse_loss": false,
-    "dsa_kernel_backend": "none",
-    "dsa_indexer_rope_interleaved": false,
-    "dsa_indexer_rotate_activation": true,
-    "dsa_indexer_scoring_relu": true,
-    "dsa_indexer_k_norm_epsilon": null,
-    "dsa_indexer_k_norm_fp32": false,
-    "csa_window_size": 128,
-    "csa_compress_ratios": null,
-    "csa_compress_rotary_base": 40000.0,
-    "csa_dense_mode": false,
-    "linear_conv_kernel_dim": 4,
-    "linear_key_head_dim": 128,
-    "linear_value_head_dim": 128,
-    "linear_num_key_heads": 16,
-    "linear_num_value_heads": 32,
-    "init_method_std": 0.02,
-    "embedding_init_method_std": null,
-    "init_model_with_meta_device": false,
-    "use_mup": false,
-    "mup_width_mult": 1.0,
-    "mup_base_hidden_size": null,
-    "mup_embedding_mult": 1.0,
-    "mup_output_mult": 1.0,
-    "mup_base_head_dim": null,
-    "mup_attn_scale_power": 1.0,
-    "apply_query_key_layer_scaling": false,
-    "disable_bf16_reduced_precision_matmul": false,
-    "use_fused_weighted_squared_relu": false,
-    "fused_residual_rmsnorm": false,
-    "use_transformer_engine_op_fuser": false,
-    "recompute_granularity": null,
-    "recompute_method": null,
-    "recompute_num_layers": null,
-    "distribute_saved_activations": false,
-    "recompute_modules": null,
-    "fp8": null,
-    "fp8_recipe": "delayed",
-    "fp8_quantizer_factory": null,
-    "fp8_margin": 0,
-    "fp8_interval": 1,
-    "fp8_amax_history_len": 1,
-    "fp8_amax_compute_algo": "most_recent",
-    "fp8_wgrad": true,
-    "fp8_output_proj": false,
-    "first_last_layers_bf16": false,
-    "num_layers_at_start_in_bf16": 1,
-    "num_layers_at_end_in_bf16": 1,
-    "use_kitchen_attention": false,
-    "kitchen_attention_backend": "sdpa",
-    "fp4": null,
-    "fp4_recipe": "nvfp4",
-    "fp4_quantizer_factory": null,
-    "high_priority_a2a_comm_stream": false,
-    "moe_shared_expert_intermediate_size": null,
-    "moe_shared_expert_gate": false,
-    "moe_shared_expert_overlap": false,
-    "use_grouped_gemm_for_shared_expert": false,
-    "moe_shared_expert_glu_interleave_size": null,
-    "moe_ffn_hidden_size": null,
-    "moe_router_topk": 2,
-    "moe_enable_routing_replay": false,
-    "moe_router_padding_for_quantization": false,
-    "moe_router_padding_for_fp8": false,
-    "moe_router_num_groups": null,
-    "moe_router_group_topk": null,
-    "moe_router_pre_softmax": false,
-    "moe_router_topk_scaling_factor": null,
-    "moe_router_score_function": "softmax",
-    "moe_router_dtype": null,
-    "moe_router_skip_muon": false,
-    "moe_router_enable_expert_bias": false,
-    "moe_router_bias_update_rate": 0.001,
-    "moe_router_quantile_balancing_ema": 0.0,
-    "moe_router_force_load_balancing": false,
-    "moe_router_force_biased": null,
-    "use_grouped_gemm_for_dense_mlp": false,
-    "moe_grouped_gemm": false,
-    "moe_use_grouped_tensor": false,
-    "moe_single_grouped_weight": false,
-    "moe_single_grouped_bias": false,
-    "moe_z_loss_coeff": null,
-    "moe_input_jitter_eps": null,
-    "moe_token_dispatcher_type": "allgather",
-    "moe_enable_deepep": false,
-    "moe_flex_dispatcher_backend": "deepep",
-    "moe_permute_fusion_into_hybridep": false,
-    "moe_hybridep_pad_uneven_dispatch_inputs": false,
-    "moe_per_layer_logging": false,
-    "moe_expert_capacity_factor": null,
-    "moe_pad_expert_input_to_capacity": false,
-    "moe_pad_experts_for_cuda_graph_inference": false,
-    "moe_token_drop_policy": "probs",
-    "moe_layer_recompute": false,
-    "moe_permute_fusion": false,
-    "moe_router_fusion": false,
-    "moe_apply_probs_on_input": false,
-    "moe_latent_size": null,
-    "moe_use_norm_before_up_proj": false,
-    "gtp_remat_opt_in_modules": [],
-    "moe_flex_dispatcher_num_sms": null,
-    "moe_deepep_num_sms": null,
-    "moe_hybridep_num_sms": null,
-    "moe_hybridep_num_blocks_permute": null,
-    "moe_hybridep_num_blocks_unpermute": null,
-    "moe_hybridep_num_sms_preprocessing": 108,
-    "moe_ncclep_zero_copy": false,
-    "moe_dispatch_fwd_dtype": "bf16",
-    "moe_combine_bwd_dtype": "bf16",
-    "moe_mlp_glu_interleave_size": null,
-    "moe_expert_rank_capacity_factor": null,
-    "linear_cp_mode": "headwise",
-    "linear_cp_layout": "zigzag",
-    "attention_cp_layout": "zigzag",
-    "enable_cuda_graph": false,
-    "cuda_graph_warmup_steps": 3,
-    "external_cuda_graph": false,
-    "cuda_graph_impl": "none",
-    "inference_cuda_graph_scope": {
-      "__enum__": [
-        "megatron.core.transformer.enums",
-        "InferenceCudaGraphScope",
-        "none"
-      ]
-    },
-    "enable_mhc_connections": false,
-    "mhc_num_residual_streams": 4,
-    "mhc_sinkhorn_iterations": 20,
-    "mhc_init_gating_factor": 0.01,
-    "use_fused_mhc": false,
-    "mhc_fused_backend": "auto",
-    "mhc_recompute_layer_num": null,
-    "clone_scatter_output_in_embedding": true,
-    "config_logger_dir": "",
-    "flash_decode": false,
-    "batch_invariant_mode": false,
-    "batch_invariant_backend": "te_native",
-    "batch_invariant_collective": "ordered",
-    "use_te_activation_func": false,
-    "symmetric_ar_type": null,
-    "nccl_all_reduce_for_prefill": false,
-    "inference_fuse_tp_communication": false,
-    "inference_disable_triton_nvls_kernels": false,
-    "inference_grouped_gemm_backend": "vllm",
-    "inference_moe_disable_fused_quant_kernels": false,
-    "inference_flashinfer_mxfp8_token_capacity": null,
-    "inference_moe_token_dispatcher_type": "nvls",
-    "mrope_section": null,
-    "is_hybrid_model": false,
-    "mamba_state_dim": 128,
-    "mamba_head_dim": 64,
-    "mamba_num_groups": 8,
-    "mamba_num_heads": null,
-    "gdp_num_householder": 3,
-    "use_mamba_mem_eff_path": true,
-    "gdp_cutedsl_kernel": false,
-    "gdp_num_chunk_states_to_recompute": 2,
-    "mlp_chunks_for_prefill": 1,
-    "mlp_chunks_for_training": 1,
-    "transformer_impl": "local",
-    "fine_grained_activation_offloading": false,
-    "offload_modules": [],
-    "min_offloaded_tensor_size": 1048576,
-    "delay_offload_until_cuda_graph": false,
-    "delta_offload_bytes_across_pp_ranks": 0,
-    "activation_offload_fraction": 1.0,
-    "moe_paged_stash": false,
-    "moe_paged_stash_page_size": 64,
-    "moe_paged_stash_buffer_size_factor_cuda": 1.1,
-    "moe_paged_stash_buffer_size_factor_cpu": 0.0,
-    "fine_grained_offloading_max_inflight_offloads": null,
-    "encoder_num_layers": 20,
-    "decoder_num_layers": null,
-    "group_query_attention": false,
-    "window_size": null,
-    "window_attn_skip_freq": null,
-    "max_position_embeddings": 2048,
-    "position_embedding_type": "rope",
-    "yarn_original_max_position_embeddings": null,
-    "yarn_beta_fast": null,
-    "yarn_beta_slow": null,
-    "yarn_correction_range_round_to_int": null,
-    "relative_attention_num_buckets": 32,
-    "relative_attention_max_distance": 128,
-    "use_rotary_position_embeddings": false,
-    "rotary_base": 10000,
-    "rotary_percent": 1.0,
-    "rotary_seq_len_interpolation_factor": null,
-    "use_rope_scaling": false,
-    "rope_scaling_factor": 8.0,
-    "no_rope_freq": null,
-    "add_position_embedding": true,
-    "make_vocab_size_divisible_by": 64,
-    "openai_gelu": false,
-    "squared_relu": false,
-    "swiglu": false,
-    "quick_geglu": false,
-    "onnx_safe": null,
-    "bert_binary_head": true,
-    "untie_embeddings_and_output_weights": true,
-    "weight_decay": 0.0,
-    "apply_wd_to_qk_layernorm": false,
-    "clip_grad": 0.0,
-    "adam_beta1": 0.8,
-    "adam_beta2": 0.95,
-    "adam_eps": 1e-10,
-    "sgd_momentum": 0.9,
-    "muon_momentum": 0.95,
-    "muon_split_qkv": true,
-    "muon_nesterov": true,
-    "muon_scale_mode": "spectral",
-    "muon_fp32_matmul_prec": "medium",
-    "muon_coefficient_type": "quintic",
-    "muon_num_ns_steps": 5,
-    "muon_tp_mode": "duplicated",
-    "muon_use_syrk": false,
-    "muon_extra_scale_factor": 1.0,
-    "muon_scalar_optimizer": "adam",
-    "lion_beta1": 0.95,
-    "lion_beta2": 0.98,
-    "no_weight_decay_cond_type": null,
-    "profile": false,
-    "profile_step_start": 10,
-    "profile_step_end": 12,
-    "use_pytorch_profiler": false,
-    "pytorch_profiler_collect_shapes": false,
-    "pytorch_profiler_collect_callstack": false,
-    "pytorch_profiler_collect_chakra": false,
-    "profile_ranks": [],
-    "record_memory_history": false,
-    "memory_snapshot_path": "snapshot.pickle",
-    "record_shapes": false,
-    "nvtx_ranges": false,
-    "micro_batch_size": 4,
-    "global_batch_size": 128,
-    "rampup_batch_size": null,
-    "step_batch_size_schedule": null,
-    "decrease_batch_size_if_needed": false,
-    "empty_unused_memory_level": 0,
-    "check_weight_hash_across_dp_replicas_interval": null,
-    "gpu_sniff_test_interval": null,
-    "train_sync_interval": null,
-    "train_iters": 26703,
-    "train_samples": null,
-    "exit_interval": null,
-    "exit_duration_in_mins": null,
-    "exit_signal_handler": false,
-    "exit_signal": {
-      "__enum__": [
-        "signal",
-        "Signals",
-        "SIGTERM"
-      ]
-    },
-    "exit_signal_handler_for_dataloader": false,
-    "exit_signal_handler_for_training": false,
-    "manual_gc": false,
-    "manual_gc_interval": 0,
-    "manual_gc_eval": true,
-    "iterations_to_skip": [],
-    "check_for_nan_in_loss_and_grad": true,
-    "check_for_large_grads": false,
-    "result_rejected_tracker_filename": null,
-    "tp_comm_overlap_cfg": null,
-    "masked_softmax_fusion": true,
-    "bias_gelu_fusion": false,
-    "bias_swiglu_fusion": true,
-    "bias_dropout_fusion": false,
-    "apply_rope_fusion": false,
-    "rope_type": null,
-    "use_flash_attn": false,
-    "optimizer": "muonh",
-    "optimizer_cpu_offload": false,
-    "optimizer_cuda_graph": false,
-    "optimizer_offload_fraction": 1.0,
-    "use_torch_optimizer_for_cpu_offload": false,
-    "overlap_cpu_optimizer_d2h_h2d": false,
-    "dump_param_to_param_group_map": null,
-    "pin_cpu_grads": true,
-    "pin_cpu_params": true,
-    "dataloader_type": "external",
-    "no_persist_layer_norm": false,
-    "gradient_accumulation_fusion": false,
-    "deprecated_use_mcore_models": false,
-    "perform_rl_step": false,
-    "rl_prompts_per_eval": 32,
-    "grpo_prompts_per_step": 32,
-    "grpo_group_size": 2,
-    "rl_generation_lag": null,
-    "rl_max_inflight_requests": null,
-    "rl_submission_granularity": "B",
-    "rl_consumption_granularity": "B",
-    "rl_durable_rollout_bank": false,
-    "rl_rollout_bank_dir": null,
-    "rl_rollout_bank_max_bytes": 0,
-    "grpo_iterations": 2,
-    "grpo_clamp_eps_lower": 0.01,
-    "grpo_clamp_eps_upper": 0.01,
-    "grpo_kl_beta": 0.001,
-    "grpo_entropy_term_weight": 0.0,
-    "grpo_filter_groups_with_same_reward": false,
-    "langrl_env_config": null,
-    "rl_default_temperature": 1.0,
-    "rl_default_top_p": 0,
-    "rl_default_top_k": -1,
-    "rl_offload_optimizer_during_inference": false,
-    "rl_kv_cache_management_mode": "persist",
-    "rl_persist_cuda_graphs": false,
-    "rl_partial_rollouts": false,
-    "rl_inference_logprobs_is_correction": false,
-    "rl_importance_sampling_truncation_coef": null,
-    "rl_use_sequence_packing": false,
-    "rl_sequence_packing_max_sequences_per_bin": 50,
-    "rl_sequence_packing_algo": "fifo",
-    "rl_training_cuda_graphs": false,
-    "rl_inference_tensor_model_parallel_size": null,
-    "rl_inference_pipeline_model_parallel_size": null,
-    "rl_inference_expert_model_parallel_size": null,
-    "rl_inference_expert_tensor_model_parallel_size": null,
-    "rl_inference_model_unified_memory_level": 0,
-    "rl_offload_inference_model_weights_when_idle": false,
-    "refit_method": "gloo",
-    "refit_execution_batch_bytes": null,
-    "rl_verify_model_weights_swap": false,
-    "rl_skip_bos_token": false,
-    "rl_profile": false,
-    "rl_profile_dir": null,
-    "rl_inference_parsers": [],
-    "seed": 42,
-    "te_rng_tracker": false,
-    "inference_rng_tracker": false,
-    "data_parallel_random_init": false,
-    "init_method_xavier_uniform": false,
-    "lr_decay_style": "linear",
-    "lr_wsd_decay_style": "linear",
-    "lr_decay_iters": null,
-    "lr_decay_samples": null,
-    "lr_wsd_decay_iters": 0,
-    "lr_wsd_decay_samples": null,
-    "lr_warmup_fraction": null,
-    "lr_warmup_iters": 0,
-    "lr_warmup_samples": 0,
-    "lr_warmup_init": 0.0,
-    "override_opt_param_scheduler": false,
-    "use_checkpoint_opt_param_scheduler": false,
-    "start_weight_decay": 0.0,
-    "end_weight_decay": 0.0,
-    "weight_decay_incr_style": "constant",
-    "lr": 0.007470638533717786,
-    "min_lr": 0.0007470638533717787,
-    "decoupled_lr": null,
-    "decoupled_min_lr": null,
-    "freeze_all_layers": false,
-    "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints",
-    "save_interval": 1000,
-    "save_params_interval": null,
-    "save_activations_interval": null,
-    "save_tokens_per_expert_interval": null,
-    "save_wgrads_interval": null,
-    "save_dgrads_interval": null,
-    "save_retain_interval": null,
-    "load": null,
-    "load_main_params_from_ckpt": false,
-    "non_persistent_save_interval": null,
-    "non_persistent_ckpt_type": null,
-    "non_persistent_global_ckpt_dir": null,
-    "non_persistent_local_ckpt_dir": null,
-    "non_persistent_local_ckpt_algo": "fully_parallel",
-    "finetune": false,
-    "pretrained_checkpoint": null,
-    "ckpt_step": null,
-    "use_checkpoint_args": false,
-    "use_mp_args_from_checkpoint_args": false,
-    "use_tokenizer_model_from_checkpoint_args": true,
-    "exit_on_missing_checkpoint": false,
-    "ckpt_format": "torch",
-    "auto_detect_ckpt_format": false,
-    "ckpt_convert_format": null,
-    "ckpt_convert_save": null,
-    "ckpt_convert_update_legacy_dist_opt_format": false,
-    "ckpt_fully_parallel_save": true,
-    "async_save": false,
-    "async_strategy": "mcore",
-    "use_persistent_ckpt_worker": false,
-    "async_ckpt_cpu_priority": 10,
-    "async_ckpt_io_priority": 3,
-    "async_ckpt_use_cpu_shm": false,
-    "ckpt_fully_parallel_load": false,
-    "ckpt_fully_parallel_load_exchange_algo": "broadcast",
-    "ckpt_fully_parallel_load_per_rank_objects": false,
-    "ckpt_fully_parallel_save_process_group": "dp",
-    "ckpt_fully_parallel_load_process_group": "dp",
-    "ckpt_assume_constant_structure": false,
-    "ckpt_pg_tensors_cache_path": null,
-    "ckpt_pg_tensors_cache_create": false,
-    "ckpt_load_validate_sharding_integrity": true,
-    "strict_fsdp_dtensor_load": true,
-    "dist_ckpt_strictness": "assume_ok_unexpected",
-    "dist_ckpt_save_pre_mcore_014": false,
-    "dist_ckpt_optim_fully_reshardable": false,
-    "distrib_optim_fully_reshardable_mem_efficient": false,
-    "save_tokenizer_assets": true,
-    "replication": false,
-    "replication_jump": null,
-    "replication_factor": 2,
-    "verify_integrity": false,
-    "no_save_optim": null,
-    "no_save_rng": null,
-    "no_load_optim": null,
-    "no_load_rng": null,
-    "override_ckpt_iteration": null,
-    "use_dist_ckpt_deprecated": false,
-    "dist_ckpt_format_deprecated": null,
-    "dist_ckpt_workers": 1,
-    "ckpt_fully_parallel_save_deprecated": false,
-    "ckpt_drop_redundant_extra_state": false,
-    "grad_reduce_in_bf16": false,
-    "loss_scale": null,
-    "initial_loss_scale": 4294967296,
-    "min_loss_scale": 1.0,
-    "loss_scale_window": 1000,
-    "hysteresis": 2,
-    "attention_softmax_in_fp32": false,
-    "accumulate_allreduce_grads_in_fp32": true,
-    "fp16_lm_cross_entropy": false,
-    "logit_dtype": null,
-    "reuse_grad_buf_for_mxfp8_param_ag": false,
-    "mamba_training_ssm_states_dtype": null,
-    "distributed_backend": "nccl",
-    "distributed_timeout_minutes": 10,
-    "align_grad_reduce": true,
-    "local_rank": 0,
-    "lazy_mpu_init": false,
-    "use_megatron_fsdp": false,
-    "use_torch_fsdp2": false,
-    "nccl_communicator_config_path": null,
-    "use_tp_pp_dp_mapping": false,
-    "use_gloo_process_groups": true,
-    "use_sharp": false,
-    "sharp_enabled_group": null,
-    "high_priority_stream_groups": [],
-    "distributed_timeout_seconds_after_init": null,
-    "flight_recorder_dump_path": null,
-    "flight_recorder_trace_buffer_size": 2000,
-    "flight_recorder_dump_on_timeout": true,
-    "flight_recorder_include_stack_trace": false,
-    "flight_recorder_include_only_active": true,
-    "flight_recorder_extra_dump_on_exec": true,
-    "disable_jit_fuser": false,
-    "decoder_first_pipeline_num_layers": null,
-    "decoder_last_pipeline_num_layers": null,
-    "pipeline_model_parallel_layout": null,
-    "num_layers_per_virtual_pipeline_stage": null,
-    "num_virtual_stages_per_pipeline_rank": null,
-    "overlap_p2p_comm": false,
-    "overlap_grad_reduce": false,
-    "ddp_num_buckets": null,
-    "ddp_bucket_size": null,
-    "ddp_pad_buckets_for_high_nccl_busbw": false,
-    "ddp_reduce_scatter_with_fp32_accumulation": false,
-    "gtp_remat_reduce_scatter_with_fp32_accumulation": false,
-    "ddp_param_name_patterns_for_fp32_local_accumulation": [],
-    "ddp_average_in_collective": false,
-    "overlap_param_gather": false,
-    "overlap_param_gather_with_optimizer_step": false,
-    "align_param_gather": false,
-    "use_distributed_optimizer": false,
-    "megatron_fsdp_version": 1,
-    "use_layer_wise_param_layout": true,
-    "nccl_ub": false,
-    "disable_symmetric_registration": false,
-    "gtp_remat_nccl_ub": false,
-    "gtp_expert_remat_nccl_ub": false,
-    "fsdp_manual_registration": false,
-    "create_all_gather_group": false,
-    "data_parallel_sharding_strategy": "optim_grads_params",
-    "expert_data_parallel_sharding_strategy": null,
-    "outer_dp_sharding_strategy": "no_shard",
-    "hfsdp_param_gather_overlap": false,
-    "gradient_reduce_div_fusion": true,
-    "fsdp_double_buffer": false,
-    "suggested_communication_unit_size": null,
-    "keep_fp8_transpose_cache": false,
-    "enable_full_sharding_in_hsdp": false,
-    "num_distributed_optimizer_instances": 1,
-    "torch_fsdp2_reshard_after_forward": true,
-    "cp_comm_type": [
-      "p2p"
-    ],
-    "max_seqlen_per_dp_cp_rank": null,
-    "hybrid_context_parallel": false,
-    "sequence_packing_scheduler": null,
-    "fake_process_group": false,
-    "eval_iters": 2,
-    "eval_interval": 100,
-    "start_eval_at_iter": null,
-    "eval_global_batch_size": 128,
-    "eval_micro_batch_size": 4,
-    "skip_train": false,
-    "test_mode": false,
-    "full_validation": false,
-    "multiple_validation_sets": false,
-    "validation_set_names": null,
-    "data_path": null,
-    "phase_transition_iterations": null,
-    "split": null,
-    "train_data_path": null,
-    "valid_data_path": null,
-    "test_data_path": null,
-    "data_args_path": null,
-    "per_split_data_args_path": null,
-    "per_dataset_sequences_path": null,
-    "dataloader_fast_cache_load": false,
-    "dataloader_defer_npy_index_mmap": false,
-    "data_cache_path": null,
-    "mmap_bin_files": true,
-    "mock_data": false,
-    "seq_length": 2048,
-    "encoder_seq_length": 2048,
-    "decoder_seq_length": null,
-    "sample_rate": 1.0,
-    "mask_prob": 0.15,
-    "short_seq_prob": 0.1,
-    "num_workers": 0,
-    "reset_position_ids": false,
-    "reset_attention_mask": false,
-    "dataloader_inter_document_masking": false,
-    "eod_mask_loss": false,
-    "create_attention_mask_in_dataloader": false,
-    "num_dataset_builder_threads": 1,
-    "object_storage_cache_path": null,
-    "mid_level_dataset_surplus": 0.005,
-    "allow_ambiguous_pad_tokens": false,
-    "fim_data": false,
-    "fim_rate": 0.5,
-    "fim_spm_rate": 0.5,
-    "fim_split_sample": null,
-    "fim_fragment_rate": null,
-    "fim_no_prefix": null,
-    "fim_prefix_token": "<fim_prefix>",
-    "fim_middle_token": "<fim_middle>",
-    "fim_suffix_token": "<fim_suffix>",
-    "fim_pad_token": "<fim_pad>",
-    "fim_eod_token": "<|endoftext|>",
-    "vocab_size": 32001,
-    "padded_vocab_size": 32064,
-    "pad_vocab_size": true,
-    "vocab_file": null,
-    "merge_file": null,
-    "vocab_extra_ids": 0,
-    "tokenizer_type": "HuggingFaceTokenizer",
-    "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a",
-    "metadata_path": null,
-    "special_tokens": null,
-    "tiktoken_pattern": null,
-    "tiktoken_num_special_tokens": 1000,
-    "tokenizer_sentencepiece_legacy": false,
-    "tokenizer_sentencepiece_ignore_extra_whitespaces": true,
-    "tokenizer_hf_no_use_fast": false,
-    "tokenizer_hf_no_include_special_tokens": false,
-    "trust_remote_code": false,
-    "null_tokenizer_eod_id": null,
-    "null_tokenizer_pad_id": -1,
-    "chat_template": null,
-    "use_gigatoken": false,
-    "adlr_autoresume": false,
-    "adlr_autoresume_interval": 1000,
-    "ict_head_size": null,
-    "biencoder_projection_dim": 0,
-    "biencoder_shared_query_context_model": false,
-    "ict_load": null,
-    "bert_load": null,
-    "titles_data_path": null,
-    "query_in_block_prob": 0.1,
-    "use_one_sent_docs": false,
-    "evidence_data_path": null,
-    "retriever_report_topk_accuracies": [],
-    "retriever_score_scaling": false,
-    "block_data_path": null,
-    "embedding_path": null,
-    "indexer_batch_size": 128,
-    "indexer_log_interval": 1000,
-    "num_classes": 1000,
-    "img_h": 224,
-    "img_w": 224,
-    "num_channels": 3,
-    "patch_dim": 16,
-    "classes_fraction": 1.0,
-    "data_per_class_fraction": 1.0,
-    "data_sharding": true,
-    "head_lr_mult": 1.0,
-    "vision_pretraining": false,
-    "vision_pretraining_type": "classify",
-    "vision_backbone_type": "vit",
-    "swin_backbone_type": "tiny",
-    "mask_type": "random",
-    "mask_factor": 1.0,
-    "iter_per_epoch": 1250,
-    "dino_local_img_size": 96,
-    "dino_local_crops_number": 10,
-    "dino_head_hidden_size": 2048,
-    "dino_bottleneck_size": 256,
-    "dino_freeze_last_layer": 1,
-    "dino_norm_last_layer": false,
-    "dino_warmup_teacher_temp": 0.04,
-    "dino_teacher_temp": 0.07,
-    "dino_warmup_teacher_temp_epochs": 30,
-    "num_experts": null,
-    "moe_layer_freq": 1,
-    "moe_use_upcycling": false,
-    "moe_router_load_balancing_type": "aux_loss",
-    "moe_aux_loss_coeff": 0.0,
-    "moe_upcycling_granularity": 1,
-    "q_lora_rank": null,
-    "kv_lora_rank": 32,
-    "qk_head_dim": 128,
-    "qk_pos_emb_head_dim": 64,
-    "v_head_dim": 128,
-    "rotary_scaling_factor": 1.0,
-    "mscale": 1.0,
-    "mscale_all_dim": 0.0,
-    "cache_mla_latents": false,
-    "mla_down_proj_fusion": false,
-    "linear_attention_freq": null,
-    "heterogeneous_layers_config_path": null,
-    "heterogeneous_layers_config_encoded_json": null,
-    "log_interval": 1,
-    "log_params_norm": false,
-    "log_throughput": false,
-    "log_progress": false,
-    "timing_log_level": 0,
-    "timing_log_option": "minmax",
-    "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard",
-    "tensorboard_log_interval": 1,
-    "tensorboard_queue_size": 1000,
-    "log_timers_to_tensorboard": true,
-    "log_loss_scale_to_tensorboard": true,
-    "log_validation_ppl_to_tensorboard": false,
-    "log_memory_to_tensorboard": false,
-    "log_memory_interval": null,
-    "log_device_memory_used": false,
-    "log_num_zeros_in_grad": false,
-    "log_max_attention_logit": false,
-    "barrier_with_L1_time": true,
-    "log_world_size_to_tensorboard": false,
-    "wandb_project": "loopTF-workspace",
-    "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm",
-    "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb",
-    "wandb_entity": "LoopTF-4-CSPs",
-    "logging_level": null,
-    "log_energy": false,
-    "moe_routing_trace_path": null,
-    "moe_routing_trace_max_training_iters": null,
-    "moe_routing_trace_capture_logits": false,
-    "moe_routing_trace_capture_hidden_states": false,
-    "moe_routing_trace_dump_weights": false,
-    "otel_enabled": false,
-    "otel_service_name": null,
-    "otel_span_groups": null,
-    "logits_save_top_k": null,
-    "logits_save_top_p": null,
-    "logits_save_top_p_min_k": 1,
-    "logits_save_dir": null,
-    "logits_save_dtype": "fp16",
-    "logits_load_dir": null,
-    "logits_load_decode_threads": 4,
-    "logits_load_prefetch_factor": 3,
-    "logits_load_msc_prefetch_depth": 2,
-    "logits_load_kd_loss_alpha": 1.0,
-    "logits_load_ignore_errors": false,
-    "log_straggler": false,
-    "straggler_ctrlr_port": 65535,
-    "straggler_minmax_count": 1,
-    "disable_straggler_on_startup": false,
-    "run_workload_inspector_server": false,
-    "inference_batch_times_seqlen_threshold": -1,
-    "max_tokens_to_oom": 12000,
-    "output_bert_embeddings": false,
-    "bert_embedder_type": "megatron",
-    "cuda_graph_modules": [],
-    "use_legacy_static_engine": false,
-    "inference_max_requests": 8,
-    "inference_max_seq_length": 2560,
-    "inference_dynamic_batching": false,
-    "inference_dynamic_batching_buffer_size_gb": 40.0,
-    "inference_dynamic_batching_paused_buffer_size_gb": null,
-    "inference_dynamic_batching_mamba_memory_ratio": null,
-    "inference_dynamic_batching_block_size": 256,
-    "inference_dynamic_batching_max_requests": null,
-    "inference_dynamic_batching_max_tokens": null,
-    "inference_dynamic_batching_num_cuda_graphs": 16,
-    "inference_dynamic_batching_track_paused_request_events": false,
-    "inference_dynamic_batching_track_generated_token_events": false,
-    "decode_only_cuda_graphs": false,
-    "inference_cuda_graph_all_prefills": false,
-    "inference_cuda_graph_max_tokens": 512,
-    "inference_dynamic_batching_unified_memory_level": 0,
-    "enable_chunked_prefill": false,
-    "num_speculative_tokens": 0,
-    "inference_dynamic_batching_enable_prefix_caching": false,
-    "inference_dynamic_batching_prefix_caching_eviction_policy": "lru",
-    "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix",
-    "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0,
-    "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0,
-    "inference_dynamic_batching_media_cache_coordinator_policy": "affinity",
-    "inference_dynamic_batching_media_cache_routing_weight": 1.0,
-    "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0,
-    "inference_dynamic_batching_allow_stale_multimodal_embeddings": false,
-    "inference_dynamic_batching_prefix_caching_mamba_gb": null,
-    "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16,
-    "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid",
-    "inference_dynamic_batching_sampling_backend": "torch",
-    "offset_sampling_seed_by_dp_rank": true,
-    "inference_dynamic_batching_async_sched_mode": "async",
-    "inference_dynamic_batching_logprobs_mode": "raw_logprobs",
-    "inference_logging_step_interval": 0,
-    "inference_text_gen_server_logging": false,
-    "inference_wandb_logging": false,
-    "inference_coordinator_port": null,
-    "mamba_inference_conv_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "mamba_inference_ssm_states_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "inference_use_synchronous_zmq_collectives": false,
-    "inference_disable_ep_consensus": false,
-    "inference_shards": null,
-    "fp4_param_gather": false,
-    "fp8_param_gather": false,
-    "te_precision_config_file": null,
-    "enable_experimental": false,
-    "spec": null,
-    "hybrid_layer_pattern": null,
-    "hybrid_override_pattern": null,
-    "yaml_cfg": null,
-    "use_precision_aware_optimizer": false,
-    "main_grads_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "exp_avg_sq_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_params_dtype": {
-      "__torch_dtype__": "float32"
-    },
-    "megatron_fsdp_main_grads_dtype": null,
-    "megatron_fsdp_grad_comm_dtype": null,
-    "megatron_fsdp_enable_fine_grained_param_gather": false,
-    "megatron_fsdp_max_pool_double_buffer": false,
-    "fsdp_db_use_persist_buf_on_alloc_fail": false,
-    "enable_one_logger": true,
-    "one_logger_project": "megatron-lm",
-    "one_logger_run_name": null,
-    "one_logger_async": false,
-    "app_tag_run_name": null,
-    "app_tag_run_version": "0.0.0",
-    "inprocess_restart": false,
-    "inprocess_max_iterations": null,
-    "inprocess_monitor_thread_interval": 1.0,
-    "inprocess_monitor_process_interval": 1.0,
-    "inprocess_progress_watchdog_interval": 1.0,
-    "inprocess_heartbeat_interval": 30,
-    "inprocess_soft_timeout": 60,
-    "inprocess_hard_timeout": 90,
-    "inprocess_heartbeat_timeout": 60,
-    "inprocess_barrier_timeout": 120,
-    "inprocess_completion_timeout": 120,
-    "inprocess_last_call_wait": 1,
-    "inprocess_termination_grace_time": 1,
-    "inprocess_granularity": "node",
-    "inprocess_active_world_size": 16,
-    "inprocess_empty_cuda_cache": false,
-    "enable_ft_package": false,
-    "calc_ft_timeouts": false,
-    "ft_num_warmup_iters": 5,
-    "error_injection_rate": 0,
-    "error_injection_type": "transient_error",
-    "rerun_mode": "validate_results",
-    "check_for_spiky_loss": false,
-    "enable_msc": false,
-    "disable_msc_deprecated": false,
-    "sft": false,
-    "sft_tokenizer_prompt_format": "nemotron-h-aligned",
-    "sft_mock_dataset_config_json": null,
-    "use_varlen_dataset": false,
-    "varlen_sbhd_validation": false,
-    "varlen_mock_dataset_config_json": null,
-    "fault_injector_ranks": null,
-    "fault_injector_num_ranks": null,
-    "fault_injector_fault_types": null,
-    "fault_injector_fault_probabilities": null,
-    "fault_injector_fault_delay": null,
-    "fault_injector_delay_start_iteration": null,
-    "fault_injector_mtti_seconds": null,
-    "fault_injector_offset_seconds": null,
-    "fault_injector_seed": null,
-    "architecture": "full_bandwidth_loop",
-    "architecture_module": [],
-    "megatron_lm_path": "/root/dl/runtime/Megatron-LM",
-    "training_recipe": "paper2604_21106",
-    "recipe_overrides": {},
-    "recipe_requested_batch_tokens": 262144,
-    "paper_reference_width": 640,
-    "paper_reference_tokens": 983315200,
-    "paper_reference_batch_tokens": 262144,
-    "paper_embedding_lr": 0.3,
-    "paper_unembedding_lr": 0.004,
-    "paper_norm_lr": 0.005,
-    "paper_other_lr": 0.005,
-    "online_data": true,
-    "online_pack_sequences": false,
-    "online_dataset": "HuggingFaceFW/fineweb-edu",
-    "online_dataset_subset": "sample-100BT",
-    "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
-    "online_source_split": "train",
-    "online_text_column": "text",
-    "online_validation_fraction": 0.001,
-    "online_split_seed": 260421106,
-    "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "online_tokenizer_batch_size": 256,
-    "online_shuffle_buffer_records": 128,
-    "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface",
-    "online_prefetch_batches": 16,
-    "online_source_mode": "local",
-    "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json",
-    "online_validation_manifest": null,
-    "online_latency_log_dir": null,
-    "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader",
-    "eval_at_start": true,
-    "final_eval_iters": 270,
-    "backbone_profile": "paper2604_vanilla",
-    "aux_mtp_horizon": 1,
-    "aux_mtp_rank": 128,
-    "aux_mtp_loss_weight": 0.3,
-    "feedback_loss_weight": 1.0,
-    "feedback_jitter_sigma": 0.0,
-    "feedback_prefix_mixin": false,
-    "feedback_pass_mixture": "3:1.0",
-    "feedback_progressive_stages": "[]",
-    "feedback_eval_passes": 3,
-    "feedback_z_loss_coefficient": 0.0,
-    "feedback_z_loss_start_fraction": 1.0,
-    "feedback_log_hidden_change": false,
-    "feedback_scalar_lr": 0.0005,
-    "feedback_scalar_weight_decay": 0.0,
-    "feedback_normuon_beta2": 0.95,
-    "depth_num_loops": 2,
-    "loop_final_adapter": "lookahead_refiner",
-    "loop_adapter_ffn_size": 5632,
-    "loop_adapter_attention_backend": "sdpa_auto",
-    "loop_adapter_checkpoint_refiner": true,
-    "adaptive_muon_moment2_method": "normuon",
-    "adaptive_muon_beta2": 0.95,
-    "adaptive_muon_eps": 1e-10,
-    "_is_global_batch_size_explicitly_specified": true,
-    "rank": 0,
-    "world_size": 16,
-    "use_dist_ckpt": false,
-    "gtp_weight_remat_size": 1,
-    "transformer_pipeline_model_parallel_size": 1,
-    "data_parallel_size": 16,
-    "virtual_pipeline_model_parallel_size": null,
-    "params_dtype": {
-      "__torch_dtype__": "bfloat16"
-    },
-    "consumed_train_samples": 1536000,
-    "skipped_train_samples": 0,
-    "consumed_valid_samples": 30976,
-    "variable_seq_lengths": false,
-    "expert_gtp_weight_remat_size": 1,
-    "use_layer_wise_distributed_optimizer": false,
-    "recipe_optimizer": {
-      "name": "muonh",
-      "matrix_lr": 0.014,
-      "scalar_lr": null,
-      "matrix_weight_decay": 0.0,
-      "scalar_weight_decay": 0.0,
-      "embedding_lr": 0.3,
-      "unembedding_lr": 0.004,
-      "norm_lr": 0.005,
-      "other_lr": 0.005,
-      "betas": [
-        0.8,
-        0.95
-      ],
-      "eps": 1e-10,
-      "normuon_momentum": 0.95,
-      "normuon_beta2": 0.95,
-      "normuon_nesterov": true,
-      "normuon_adjust_lr": "spectral_norm",
-      "max_grad_norm": null
-    },
-    "recipe_schedule": {
-      "kind": "linear",
-      "warmup_steps": 0,
-      "cooldown_fraction": 0.0,
-      "cooldown_style": "linear",
-      "final_lr_fraction": 0.1,
-      "z_loss_coefficient": 0.0,
-      "adamc_weight_decay": false
-    },
-    "paper_base_matrix_lr": 0.014,
-    "paper_transfer": {
-      "__dataclass__": [
-        "loop_transformer_backends.recipes",
-        "Paper2604Transfer",
-        {
-          "width": 1408,
-          "total_tokens": 7000031232,
-          "batch_tokens": 262144,
-          "reference_width": 640,
-          "reference_tokens": 983315200,
-          "reference_batch_tokens": 262144
-        }
-      ]
-    },
-    "training_recipe_metadata": {
-      "recipe": "paper2604_21106",
-      "width": 1408,
-      "total_tokens": 7000031232,
-      "batch_tokens": 262144,
-      "reference_width": 640,
-      "reference_tokens": 983315200,
-      "reference_batch_tokens": 262144,
-      "base_lrs": {
-        "matrix": 0.014,
-        "embedding": 0.3,
-        "unembedding": 0.004,
-        "norm": 0.005,
-        "other": 0.005
-      },
-      "peak_lrs": {
-        "matrix": 0.007470638533717786,
-        "embedding": 0.2022599587389726,
-        "unembedding": 0.0026967994498529684,
-        "norm": 0.00337099931231621,
-        "other": 0.00337099931231621
-      },
-      "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor",
-      "tied_embedding_policy": "unembedding (one AdamW update)",
-      "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)",
-      "weight_decay": 0.0,
-      "warmup_steps": 0,
-      "final_lr_fraction": 0.1,
-      "optimizer": {
-        "name": "muonh",
-        "matrix_lr": 0.014,
-        "scalar_lr": null,
-        "matrix_weight_decay": 0.0,
-        "scalar_weight_decay": 0.0,
-        "embedding_lr": 0.3,
-        "unembedding_lr": 0.004,
-        "norm_lr": 0.005,
-        "other_lr": 0.005,
-        "betas": [
-          0.8,
-          0.95
-        ],
-        "eps": 1e-10,
-        "normuon_momentum": 0.95,
-        "normuon_beta2": 0.95,
-        "normuon_nesterov": true,
-        "normuon_adjust_lr": "spectral_norm",
-        "max_grad_norm": null
-      },
-      "schedule": {
-        "kind": "linear",
-        "warmup_steps": 0,
-        "cooldown_fraction": 0.0,
-        "cooldown_style": "linear",
-        "final_lr_fraction": 0.1,
-        "z_loss_coefficient": 0.0,
-        "adamc_weight_decay": false
-      },
-      "group_weight_decay": {
-        "matrix": 0.0,
-        "embedding": 0.0,
-        "unembedding": 0.0,
-        "norm": 0.0,
-        "other": 0.0
-      },
-      "recipe_overrides": {},
-      "requested_batch_tokens": 262144,
-      "sequence_length": 2048,
-      "batch_assumptions": {
-        "global_batch_tokens": 262144,
-        "sequence_length": 2048,
-        "round_up": false
-      },
-      "lr_transfer": "mup_hyperp_released_source",
-      "assumptions": [],
-      "backend_kernel_assumptions": {},
-      "paper_settings_overridden": false,
-      "model": {
-        "backbone_profile": "paper2604_vanilla",
-        "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-        "width_scale": 22,
-        "hidden_size": 1408,
-        "num_layers": 20,
-        "head_dim": 128,
-        "num_attention_heads": 11,
-        "num_kv_heads": 11,
-        "ffn_hidden_size": 5632,
-        "activation": "squared_relu",
-        "attention": "full_causal",
-        "attention_output_gate": false,
-        "qk_norm": "functional_rmsnorm_after_rope",
-        "norm_epsilon": 1e-06,
-        "model_level_norms": [
-          "embedding",
-          "recur",
-          "final"
-        ],
-        "rope_base": 10000.0,
-        "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-        "bias": false,
-        "dropout": 0.0,
-        "residual_multiplier": 1.0,
-        "residual_and_norm_scale_dtype": "float32",
-        "projection_dtype": "bfloat16 (production)",
-        "tied_embeddings": false,
-        "vocab_size": 32001,
-        "padded_vocab_size": 32064,
-        "logit_softcap": 15.0,
-        "crop_padding_before_loss": true,
-        "initialization": {
-          "embedding_normal_std": 1.0,
-          "head_normal_std": 0.001,
-          "matrix_uniform_bound": "sqrt(3/fan_in)",
-          "norm_scale": 1.0
-        },
-        "parameter_counts": {
-          "non_embedding_matrix_formula": 475791360,
-          "non_embedding_paper_N": 475851904,
-          "total_trainable": 653383808,
-          "feedback_parameters": 3967744,
-          "loop_adapter_parameters": 83271936
-        },
-        "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-        "execution_architecture": "full_bandwidth_loop",
-        "objective": "latent_feedback_causal_lm",
-        "feedback_enabled": true,
-        "train_passes": 3,
-        "eval_passes": 3,
-        "feedback_pass_mixture": {
-          "3": 1.0
-        },
-        "feedback_progressive_stages": [],
-        "depth_num_loops": 2,
-        "logical_layers_per_pass": 40,
-        "shared_layer_weights": true,
-        "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-        "loop_final_adapter": "lookahead_refiner",
-        "loop_adapter": {
-          "ffn_size": 5632,
-          "state_input": "second-loop hidden states after norm_recur and norm_final",
-          "terminal_norms": true,
-          "attention_backend": "sdpa_auto",
-          "checkpoint_refiner": true,
-          "bank_layers": [
-            0,
-            10,
-            19
-          ],
-          "bank_source": "first depth loop, fresh on each feedback pass",
-          "weight_sharing": "one adapter shared across feedback passes"
-        },
-        "block_application_multiplier_per_pass": 2.15,
-        "block_application_multiplier": 6.449999999999999
-      }
-    },
-    "backbone_metadata": {
-      "backbone_profile": "paper2604_vanilla",
-      "source_commit": "122366472a31e57982a2877eeee10bb115477713",
-      "width_scale": 22,
-      "hidden_size": 1408,
-      "num_layers": 20,
-      "head_dim": 128,
-      "num_attention_heads": 11,
-      "num_kv_heads": 11,
-      "ffn_hidden_size": 5632,
-      "activation": "squared_relu",
-      "attention": "full_causal",
-      "attention_output_gate": false,
-      "qk_norm": "functional_rmsnorm_after_rope",
-      "norm_epsilon": 1e-06,
-      "model_level_norms": [
-        "embedding",
-        "recur",
-        "final"
-      ],
-      "rope_base": 10000.0,
-      "rope_convention": "split_half_negative_angle_bf16_cos_sin",
-      "bias": false,
-      "dropout": 0.0,
-      "residual_multiplier": 1.0,
-      "residual_and_norm_scale_dtype": "float32",
-      "projection_dtype": "bfloat16 (production)",
-      "tied_embeddings": false,
-      "vocab_size": 32001,
-      "padded_vocab_size": 32064,
-      "logit_softcap": 15.0,
-      "crop_padding_before_loss": true,
-      "initialization": {
-        "embedding_normal_std": 1.0,
-        "head_normal_std": 0.001,
-        "matrix_uniform_bound": "sqrt(3/fan_in)",
-        "norm_scale": 1.0
-      },
-      "parameter_counts": {
-        "non_embedding_matrix_formula": 475791360,
-        "non_embedding_paper_N": 475851904,
-        "total_trainable": 653383808,
-        "feedback_parameters": 3967744,
-        "loop_adapter_parameters": 83271936
-      },
-      "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.",
-      "execution_architecture": "full_bandwidth_loop",
-      "objective": "latent_feedback_causal_lm",
-      "feedback_enabled": true,
-      "train_passes": 3,
-      "eval_passes": 3,
-      "feedback_pass_mixture": {
-        "3": 1.0
-      },
-      "feedback_progressive_stages": [],
-      "depth_num_loops": 2,
-      "logical_layers_per_pass": 40,
-      "shared_layer_weights": true,
-      "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm",
-      "loop_final_adapter": "lookahead_refiner",
-      "loop_adapter": {
-        "ffn_size": 5632,
-        "state_input": "second-loop hidden states after norm_recur and norm_final",
-        "terminal_norms": true,
-        "attention_backend": "sdpa_auto",
-        "checkpoint_refiner": true,
-        "bank_layers": [
-          0,
-          10,
-          19
-        ],
-        "bank_source": "first depth loop, fresh on each feedback pass",
-        "weight_sharing": "one adapter shared across feedback passes"
-      },
-      "block_application_multiplier_per_pass": 2.15,
-      "block_application_multiplier": 6.449999999999999
-    },
-    "evaluation_metadata": {
-      "eval_at_start": true,
-      "initial_iters": 2,
-      "intermediate_iters": 2,
-      "final_validation_iters": 270,
-      "test_iters": 0,
-      "global_batch_size": 128,
-      "sequence_length": 2048,
-      "intermediate_input_tokens": 524288,
-      "final_validation_sequences": 34560,
-      "final_validation_input_tokens": 70778880,
-      "validation_iterator": "continues_existing_held_out_iterator"
-    },
-    "iteration": 0,
-    "num_floating_point_operations_so_far": 0,
-    "do_train": 1,
-    "do_valid": 1,
-    "do_test": 0,
-    "curr_iteration": 11999
-  },
-  "non_tensor_state": {
-    "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null,
-    "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null,
-    "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null,
-    "output_layer._extra_state": null,
-    "feedback.state_proj._extra_state": null,
-    "feedback.token_gate_proj._extra_state": null
-  },
-  "tensor_parallel_size": 1,
-  "pipeline_parallel_size": 1,
-  "training_tokens": 3145728000,
-  "training_budget_tokens": 7000031232,
-  "tokenizer": {
-    "repository": "NousResearch/Llama-2-7b-hf",
-    "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d",
-    "added_tokens": [
-      "<|assistant_end|>"
-    ],
-    "active_vocab_size": 32001,
-    "padded_vocab_size": 32064
-  },
-  "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README."
-}