diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/README.md b/loop_ifm_freshkv_7b_20260920-213933/step_26703/README.md deleted file mode 100644 index 0919bc132bf38fb8a36f32a6a1059a916a09d7f4..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_freshkv_7b_20260920-213933 / step_26703 - -Native fresh-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Final validation metrics are recorded from this run in config.json. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 recomputes fresh KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/config.json b/loop_ifm_freshkv_7b_20260920-213933/step_26703/config.json deleted file mode 100644 index 9af4b2ae3bf3407469b7527e5eaa49d2ac53919d..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/config.json +++ /dev/null @@ -1,1431 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_freshkv_7b_20260920-213933", - "iteration": 26703, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_freshkv_7b_20260920-213933", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_freshkv_7b_20260920-213933/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": false, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 3417984, - "skipped_train_samples": 0, - "consumed_valid_samples": 68608, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": false, - "depth_kv_policy": "recompute_each_loop", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": false, - "depth_kv_policy": "recompute_each_loop", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 26702 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 7000031232, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": false, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_freshkv_7b_20260920-213933", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README.", - "final_validation": { - "eval/final_val/pass_1_ce": 2.293280601501465, - "eval/final_val/pass_1_ppl": 9.907386613462691, - "eval/final_val/pass_2_ce": 2.2767744064331055, - "eval/final_val/pass_2_ppl": 9.74519561796341, - "eval/final_val/pass_3_ce": 2.276322364807129, - "eval/final_val/pass_3_ppl": 9.74079137941538, - "eval/final_val/final_ce": 2.276322364807129, - "eval/final_val/final_ppl": 9.74079137941538, - "eval/final_val/pass_count": 3, - "eval/checkpoint_step": 26703, - "eval/is_final": true, - "eval/final_val/num_sequences": 34560, - "eval/final_val/num_tokens": 70778880, - "eval/final_val/num_input_tokens": 70778880, - "eval/final_val/dataset_id": "config-cb8c1cc82eb53c5c" - } -} diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/launch_config.env b/loop_ifm_freshkv_7b_20260920-213933/step_26703/launch_config.env deleted file mode 100644 index cd5b6f1198f515ef8bb23a327b61af56e3451420..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=false -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_freshkv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,freshkv diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/manifest.json b/loop_ifm_freshkv_7b_20260920-213933/step_26703/manifest.json deleted file mode 100644 index 807bf9744aa5d6cf5d6c85499d162638f115846e..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3416, - "sha256": "4dfeba7349838f5bba2b8948625553c3e3dade8bc5a1a6d3f38de467ae752ab1" - }, - "config.json": { - "size": 54867, - "sha256": "f44859fba61361b4fd88175d182ec90e92cf3cdaa34ccc188168724d1bb16185" - }, - "launch_config.env": { - "size": 534, - "sha256": "a5b0afb57b1d34b2c1cee722ff79fc6eeb4b7b246493f29fb0414397be23e7bd" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "5e819f113b4014446733cb8bda9b00bbb550aa25cd5b97a995f40a165d2d3286" - }, - "training_recipe.json": { - "size": 5092, - "sha256": "38b5c1786244228c996724ea6ee630898f5bcb898774abc0e3aadff1326dafa9" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/model.safetensors b/loop_ifm_freshkv_7b_20260920-213933/step_26703/model.safetensors deleted file mode 100644 index 03fa50eaf049a71cfbee496cc0bd1d8b291e9223..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5e819f113b4014446733cb8bda9b00bbb550aa25cd5b97a995f40a165d2d3286 -size 1310894448 diff --git a/loop_ifm_freshkv_7b_20260920-213933/step_26703/training_recipe.json b/loop_ifm_freshkv_7b_20260920-213933/step_26703/training_recipe.json deleted file mode 100644 index 8fa9e18288a2befc5b7b45a8d9c780e5c1ec15a5..0000000000000000000000000000000000000000 --- a/loop_ifm_freshkv_7b_20260920-213933/step_26703/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": false, - "depth_kv_policy": "recompute_each_loop", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_12000/README.md deleted file mode 100644 index ea6613ec1bbda88a8c002b5f5fa648a2c3eb5b66..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_12000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_12000/config.json deleted file mode 100644 index 3c4e9b9f6db3a0b06728414d60fdcd248688b1ce..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 12000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 1536000, - "skipped_train_samples": 0, - "consumed_valid_samples": 30976, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 11999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 3145728000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_12000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_12000/manifest.json deleted file mode 100644 index fdc89ad4186ecc007f1ac51055e80511bf3c783d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3451, - "sha256": "973a0d9814fec90cc99260e9ff343ca96b36b0c54475798024856f4f61ca0cb6" - }, - "config.json": { - "size": 54156, - "sha256": "c21201abf80edb8a3cf18ffed6c290c2e17d829441913909c4d112128682b38f" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "d153a3f096756906756fdb7fbb5b0f491b7d9cf2ec448876f0d2a6b1e386f811" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_12000/model.safetensors deleted file mode 100644 index b456b9e781bc3dde2dd395e26a9d0664235c13af..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d153a3f096756906756fdb7fbb5b0f491b7d9cf2ec448876f0d2a6b1e386f811 -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_12000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_12000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_12000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_16000/README.md deleted file mode 100644 index 99734d33f105fa63e1c4eae6e0b2b5808a458d06..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_16000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 16000, consumed 4,194,304,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_16000/config.json deleted file mode 100644 index 0e2ba321b745d3ed4785856b450f1d284c7aef4c..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 16000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 2048000, - "skipped_train_samples": 0, - "consumed_valid_samples": 41216, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 15999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 4194304000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_16000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_16000/manifest.json deleted file mode 100644 index 6ed884c6ac228f4062c91cebf7b4925f60bb08fc..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3451, - "sha256": "037e15d0f02ba5fa5205be337a7c0207e604e4f540bd0e320229849b16bfe9e2" - }, - "config.json": { - "size": 54156, - "sha256": "7e47b2b81c764d67cb70d8f06c2ec10dc909421c641aedeb06509b7a6ecf3752" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "7b6362546d56c93fc05ad7c5285b7a90e0b203a7dc022906efc694a3b703b7b5" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_16000/model.safetensors deleted file mode 100644 index e958625832f550f2121d55d48dc67743c1aa1cee..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7b6362546d56c93fc05ad7c5285b7a90e0b203a7dc022906efc694a3b703b7b5 -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_16000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_16000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_16000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_20000/README.md deleted file mode 100644 index 0b116c7f520690748e3b1ba81d138640db5d7165..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_20000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 20000, consumed 5,242,880,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_20000/config.json deleted file mode 100644 index e88e25667aed37e39fbf9372e0887d30abd02294..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 20000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 2560000, - "skipped_train_samples": 0, - "consumed_valid_samples": 51456, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 19999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 5242880000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_20000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_20000/manifest.json deleted file mode 100644 index c77265dc747afae165655afb21d420bcb72131f4..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3451, - "sha256": "10e6715718b11885ba343029d01e3f3d562da7e03c71eebcae8118e2728d80c0" - }, - "config.json": { - "size": 54156, - "sha256": "9ed6d74bdbb56315a74a534022f6bc02c65c519c19c59fb50e622c2ebce344cf" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "35fede4757419e4449384de7e64a0f4a8e92305937fdf6dde9cc00c3cbf6b175" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_20000/model.safetensors deleted file mode 100644 index 648de48d8f1174300390d4a2fc97def4d8b82221..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:35fede4757419e4449384de7e64a0f4a8e92305937fdf6dde9cc00c3cbf6b175 -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_20000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_20000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_20000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_24000/README.md deleted file mode 100644 index 9e764e1e0f2adedad23aa24eca6228aaae9817f4..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_24000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 24000, consumed 6,291,456,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_24000/config.json deleted file mode 100644 index 46c0ea5684a5962cc788c14ccddae991572e0d10..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 24000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 3072000, - "skipped_train_samples": 0, - "consumed_valid_samples": 61696, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 23999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 6291456000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_24000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_24000/manifest.json deleted file mode 100644 index 4891c6a37cfd606044234aa543564ef07768e268..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3451, - "sha256": "027f6622ec6f3d649eab9ab5e6f696bc166fc7b354bb08676f2c984f8c1ab483" - }, - "config.json": { - "size": 54156, - "sha256": "2d92ea08687ce5bff6bec73c14a3562c062769c291d1d4031380adc88c3be103" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "dad05a5a544259a3e41d069b609790a4d38d3c1c955c89eb8b40cb3a40433f01" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_24000/model.safetensors deleted file mode 100644 index f54eb6dc62109d7b52880ee05f7e60edbfc1ec77..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dad05a5a544259a3e41d069b609790a4d38d3c1c955c89eb8b40cb3a40433f01 -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_24000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_24000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_24000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_26703/README.md deleted file mode 100644 index 07ed2edbc69950c660c60b37c62cae51c02f3d8e..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_26703 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_26703/config.json deleted file mode 100644 index c7e95945641925c347fe705fecc5dad1a7690695..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/config.json +++ /dev/null @@ -1,1431 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 26703, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 3417984, - "skipped_train_samples": 0, - "consumed_valid_samples": 68608, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 26702 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 7000031232, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README.", - "final_validation": { - "eval/final_val/pass_1_ce": 2.304596424102783, - "eval/final_val/pass_1_ppl": 10.020133551919583, - "eval/final_val/pass_2_ce": 2.280852794647217, - "eval/final_val/pass_2_ppl": 9.785021466348496, - "eval/final_val/pass_3_ce": 2.279691696166992, - "eval/final_val/pass_3_ppl": 9.773666686079672, - "eval/final_val/final_ce": 2.279691696166992, - "eval/final_val/final_ppl": 9.773666686079672, - "eval/final_val/pass_count": 3, - "eval/checkpoint_step": 26703, - "eval/is_final": true, - "eval/final_val/num_sequences": 34560, - "eval/final_val/num_tokens": 70778880, - "eval/final_val/num_input_tokens": 70778880, - "eval/final_val/dataset_id": "config-cb8c1cc82eb53c5c" - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_26703/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_26703/manifest.json deleted file mode 100644 index faaa58fb369f439b55a1f689a76db9f89b1163df..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3451, - "sha256": "a2d145dd6896a99b66a0d8e602b2c4b0f3e2caf72f2d528cffbc1fe5603ff7fb" - }, - "config.json": { - "size": 54888, - "sha256": "7c0746b2f1cca12d677f6dbec74634baa42401883d2890bd963077a3ff4059b0" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "8d961dacfd1cf67f11536a4eba6c73df8e822788b840fd99ed10d73330b5d976" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_26703/model.safetensors deleted file mode 100644 index 1d391cd9f240eeb4a7ceb96d123ee75f5d194014..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8d961dacfd1cf67f11536a4eba6c73df8e822788b840fd99ed10d73330b5d976 -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_26703/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_26703/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_26703/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_4000/README.md deleted file mode 100644 index e20fdde3bdec0bbcaed7782f56fa1db041027ac5..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_4000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 4000, consumed 1,048,576,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_4000/config.json deleted file mode 100644 index f9cce35b9301cdc95eb126f71e46e625781f0aa7..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 4000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 512000, - "skipped_train_samples": 0, - "consumed_valid_samples": 10496, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 3999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 1048576000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_4000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_4000/manifest.json deleted file mode 100644 index f4863647e158e726a1338d88560a7b328d295dcb..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3449, - "sha256": "4eb470c9b5019a8ae3131b705bde16633ccf14ccc1e5fb9434c7c49c8fe4d57e" - }, - "config.json": { - "size": 54153, - "sha256": "f60eda6d58a8aae9a0eab9058cb334d821090fbb3bdc3c01a46537543070affc" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "f0c51d09d782f0b8b10451a6f6f85de49ca4baa7ae0777f84de5955d0ba01d0a" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_4000/model.safetensors deleted file mode 100644 index c0c5060d287a42fb8c2c4dbd0e1efa52650a51f2..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f0c51d09d782f0b8b10451a6f6f85de49ca4baa7ae0777f84de5955d0ba01d0a -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_4000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_4000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_4000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/README.md b/loop_ifm_sharekv_7b_20260920-153012/step_8000/README.md deleted file mode 100644 index 7dc9dc991f40269f9f1ba8c4a3356632e1b41d41..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# loop_ifm_sharekv_7b_20260920-153012 / step_8000 - -Native shared-KV full-bandwidth loop K3/L2 with IFM diagonal raw-embedding writes and a terminal adapter, 655,369,088 parameters. Checkpoint step 8000, consumed 2,097,152,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/414cc438e67f921df1fcd218dcd738b88c83c2e1 (repository access may be required). The training revision is recorded in config.json; the saved code.diff is empty. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. IFM injects raw token embeddings before both depth loops with shared projection/gating parameters. Loop2 reuses connected first-loop KV. Preserve all IFM tensors and settings when loading; a non-IFM model is incompatible. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/config.json b/loop_ifm_sharekv_7b_20260920-153012/step_8000/config.json deleted file mode 100644 index 4d3fea7d8fbe6f0782b5719f2bdaf68e4f0fdfaf..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/config.json +++ /dev/null @@ -1,1414 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "loop_ifm_sharekv_7b_20260920-153012", - "iteration": 8000, - "checkpoint_version": 3.0, - "training_code_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "training_base_revision": "414cc438e67f921df1fcd218dcd738b88c83c2e1", - "source_provenance": "Training used training_code_revision with an empty saved code.diff.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 655369088, - "trainable": 655369088 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 8, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/checkpoints", - "save_interval": 4000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": 4000, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 8, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "loop_ifm_sharekv_7b_20260920-153012", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/research/loop_ifm/loop_ifm_sharekv_7b_20260920-153012/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_layers": null, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "depth_share_kv": true, - "loop_input_write": "ifm_diagonal", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 1024000, - "skipped_train_samples": 0, - "consumed_valid_samples": 20736, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 7999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 2097152000, - "training_budget_tokens": 7000031232, - "ifm_diagonal": { - "source": "raw_token_embeddings", - "placement": "before_both_depth_loops", - "shared_projection_and_gating": true - }, - "depth_share_kv": true, - "effective_launch_overrides": { - "NNODES": 2, - "MICRO_BATCH_SIZE": 8, - "GLOBAL_BATCH_SIZE": 128, - "RUN_NAME": "loop_ifm_sharekv_7b_20260920-153012", - "SAVE_INTERVAL": 4000, - "SAVE_RETAIN_INTERVAL": 4000, - "ENABLE_RECOMPUTE": false, - "LOOP_ADAPTER_CHECKPOINT_REFINER": false - }, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/launch_config.env b/loop_ifm_sharekv_7b_20260920-153012/step_8000/launch_config.env deleted file mode 100644 index d83ee840b741f1599efbf9732157f62d8c3ba87d..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/launch_config.env +++ /dev/null @@ -1,12 +0,0 @@ -# IFM raw-token input injection before every depth loop; fresh 16-GPU arm. -source "$(dirname "${BASH_SOURCE[0]}")/476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env" -LOOP_INPUT_WRITE=ifm_diagonal -DEPTH_SHARE_KV=true -NNODES=2 -MICRO_BATCH_SIZE=8 -ENABLE_RECOMPUTE=false -LOOP_ADAPTER_CHECKPOINT_REFINER=false -SAVE_RETAIN_INTERVAL=4000 -RUN_NAME=paper2604_loop2_476m_7b_fbw_k3_adapter_ifm_sharekv -WANDB_RUN_GROUP=loop2_adapter_ifm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,K3,lookahead_refiner,ifm_diagonal,sharekv diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/manifest.json b/loop_ifm_sharekv_7b_20260920-153012/step_8000/manifest.json deleted file mode 100644 index 08c05e65afedf1224da2111be3c2462a748e7db3..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3449, - "sha256": "76a7963c72a7dfb46270f7e2d4c8b1e730ef346062d0624d5aaad3a9e5b7c8d1" - }, - "config.json": { - "size": 54154, - "sha256": "358cd038eea825c018625fbaace647c0b7843987cd9e8bb77f235629c06cf644" - }, - "launch_config.env": { - "size": 533, - "sha256": "8573d72be7041dca69dea2d3d6cdaf7fc8e8ebf6b48f28dd0860ddaf806e4b72" - }, - "model.safetensors": { - "size": 1310894448, - "sha256": "c3674b08899fb3e3b7c10f1366f998f5ce7f148df52f05060f1fc5743b2972ab" - }, - "training_recipe.json": { - "size": 5102, - "sha256": "9d83cdc447b73c02c4b65e3e41fe8e9dcd215d22a90b15b403904203af0df61d" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/model.safetensors b/loop_ifm_sharekv_7b_20260920-153012/step_8000/model.safetensors deleted file mode 100644 index 90306d0689aa3069f51af260548645cb2ab7c0af..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c3674b08899fb3e3b7c10f1366f998f5ce7f148df52f05060f1fc5743b2972ab -size 1310894448 diff --git a/loop_ifm_sharekv_7b_20260920-153012/step_8000/training_recipe.json b/loop_ifm_sharekv_7b_20260920-153012/step_8000/training_recipe.json deleted file mode 100644 index d87ac5e808400db47c6eec4d002e21a4ac5e4fd8..0000000000000000000000000000000000000000 --- a/loop_ifm_sharekv_7b_20260920-153012/step_8000/training_recipe.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 655369088, - "feedback_parameters": 3967744, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/README.md deleted file mode 100644 index b17a06a29cc00a580cd3afb1d604e8dda53c2090..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_12000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 12000, consumed 3,145,728,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/config.json deleted file mode 100644 index 39fde31bd1a4a5784373bf49c693ef1b67e3de59..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 12000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 1536000, - "skipped_train_samples": 0, - "consumed_valid_samples": 30976, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 11999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 3145728000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/manifest.json deleted file mode 100644 index f7e8e95dcc14664f98bb0bb186cf37641b010c11..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3196, - "sha256": "28a92e011ba523d576795efb1f5e3bc61650acfe1db892104367a0c9503703fa" - }, - "config.json": { - "size": 52755, - "sha256": "15ca9b972a75ca235b7d01ef7fec750bdbd78ea4bbe7efd324fc5960f2d0ef4c" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "72d7cc578222a8bd2b4153b2166c92f72bd21ce0ed4dcced59b6e3f69d0ecd5f" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/model.safetensors deleted file mode 100644 index 67a534605e5dfe897b9a553d7a3df43f316dafe8..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:72d7cc578222a8bd2b4153b2166c92f72bd21ce0ed4dcced59b6e3f69d0ecd5f -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_12000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/README.md deleted file mode 100644 index 823b9514767ae355e7c08bc82b0fc8027c583b3c..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_16000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 16000, consumed 4,194,304,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/config.json deleted file mode 100644 index 07fc29330b6e56c0e54c68f95ed2461c4017c0bb..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 16000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 2048000, - "skipped_train_samples": 0, - "consumed_valid_samples": 41216, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 15999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 4194304000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/manifest.json deleted file mode 100644 index 33ea3a71ec6037a279b832e8831541ccfdc35eb6..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3196, - "sha256": "2b40ea004b62f3057e2e6b119ab0f01bf517823a4f94918d034790b58f45256b" - }, - "config.json": { - "size": 52755, - "sha256": "2126d395ce3b0f598fb82be2c7ec700df4554e83bce91d1b09f0d712d8e2566a" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "25ab5937c820c0249189f1ca9f8b9b9f8afeca5593049a59d8e9736d5e4c64a6" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/model.safetensors deleted file mode 100644 index 7bc1345380b436fab9fb9bd35f7edccab5178f26..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:25ab5937c820c0249189f1ca9f8b9b9f8afeca5593049a59d8e9736d5e4c64a6 -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_16000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/README.md deleted file mode 100644 index 34a8f7e8ad9f0930df11f681030b0cfb60ebd0c2..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_20000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 20000, consumed 5,242,880,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/config.json deleted file mode 100644 index 55a1b10f849761aad9ddab0fded626b67f4543b8..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 20000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 2560000, - "skipped_train_samples": 0, - "consumed_valid_samples": 51456, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 19999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 5242880000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/manifest.json deleted file mode 100644 index 421b10816cf7b0aa99610eef1e00d811e05fb93a..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3196, - "sha256": "4c8b780dd7ea9aa151c584863746a3cd7c1528521930170f00fc38c3232aab17" - }, - "config.json": { - "size": 52755, - "sha256": "1eb819a8153ec270e91b693d4f00052bba80684aa01e82543f00154e55527539" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "cafce13be0cffe8d99675158c76c5abfa8f97abc84c3e96d072b1fa7d35fac16" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/model.safetensors deleted file mode 100644 index af0cd12911986d1f9a875f56f5c59ad002ad1b65..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cafce13be0cffe8d99675158c76c5abfa8f97abc84c3e96d072b1fa7d35fac16 -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_20000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/README.md deleted file mode 100644 index f193f2eb47ee2adf8b4bb28bf5f9f48e722c5a5a..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_24000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 24000, consumed 6,291,456,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/config.json deleted file mode 100644 index b67a9a56d7ef6269e230a6344844992f5f97d468..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 24000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 3072000, - "skipped_train_samples": 0, - "consumed_valid_samples": 61696, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 23999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 6291456000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/manifest.json deleted file mode 100644 index 05308349be1e96525f7e6c70f1b58a45faa525a5..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3196, - "sha256": "b3ad9dc539f73b5bb62e4554bc92aa69b078f69cf3f8cda25e150d301cae1a22" - }, - "config.json": { - "size": 52755, - "sha256": "9912b229120fae40b2d6e97b7bae0d1c066978d66c1363842c1217fe2fb7ff66" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "6024678711b43017688d627f4f0af05e3204144bef95881d95e311e0200756f9" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/model.safetensors deleted file mode 100644 index 7fbdd482c9bfd537e4e9cf70ab16b6531e5d7099..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6024678711b43017688d627f4f0af05e3204144bef95881d95e311e0200756f9 -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_24000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/README.md deleted file mode 100644 index dc8367d0d016f129f1d2ae29d52594747d1f21b5..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_26703 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 26703, consumed 7,000,031,232 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/config.json deleted file mode 100644 index 6b1fc4d17f09c2de87ad4f2fb831d4ee53abff59..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 26703, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 3417984, - "skipped_train_samples": 0, - "consumed_valid_samples": 68608, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 26702 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 7000031232, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/manifest.json deleted file mode 100644 index 9157ebbde53db2336d76c2e6172595a91e6a91a4..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3196, - "sha256": "c065cd0d69e75ddbbdd7f1b9f9b27bd2672e0981d3ec81fcfed85b2529bf1515" - }, - "config.json": { - "size": 52755, - "sha256": "47b0c2c7dc681ffc73ef94399be8ee80c22c717b39a6e72d0b03e7d5bc852ec9" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "a087d9124c2e0f8a7fd2fcda9787c6a1743c0994f7623cee29ae7ce0c25a505e" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/model.safetensors deleted file mode 100644 index ce40f43c6d549ac0a9ae6a46c48d9d3c29671d16..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a087d9124c2e0f8a7fd2fcda9787c6a1743c0994f7623cee29ae7ce0c25a505e -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_26703/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/README.md deleted file mode 100644 index f2b6f4fa49883adc6a10eba613bfc8f1d92a00fe..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_4000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 4000, consumed 1,048,576,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/config.json deleted file mode 100644 index 6d2252484f1bac885cafd93129f3d21eb02e1cb5..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 4000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 512000, - "skipped_train_samples": 0, - "consumed_valid_samples": 10496, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 3999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 1048576000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/manifest.json deleted file mode 100644 index dd4b4edfc52b461c885d9da4ef38f730a0096340..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3194, - "sha256": "81480298e0e997b7a00e7087510acedd826c4aa45c9e4ba0072b7bc8724b8d29" - }, - "config.json": { - "size": 52752, - "sha256": "15834ea04c075d717597e0190c32e5f7389eb27597e2d2a671d3162ef3319c44" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "91c53eaf3943b0c4f0ce834954784d540f4f103cccc74fa95eca43e78edd2afd" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/model.safetensors deleted file mode 100644 index de7519f5f835c3daed4d1352f9398a855c3e4e0c..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:91c53eaf3943b0c4f0ce834954784d540f4f103cccc74fa95eca43e78edd2afd -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_4000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/README.md deleted file mode 100644 index 5d9c646e0cd6420148e623f4b7997eae492e4d02..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm / step_8000 - -Native full-bandwidth loop K3/L2 normalized-state refiner model, 653,383,808 parameters. Checkpoint step 8000, consumed 2,097,152,000 input tokens. The full run finished at 26,703 steps with exit 0. Earlier checkpoints are intermediate weights; no final evaluation metric is asserted here. - -Training snapshot source exactly matches https://github.com/FFishy-git/loopTF-workspace/tree/8ce34798f6137fdaa27e6dfccc9323b506142b83 (repository access may be required). The original dirty base revision is recorded separately in config.json. Megatron-LM revision ed6ac270bacb6a4258e1c5b4a911bcf0fd3e4d9b; use the project's training dependencies. Export verification used Python 3.13.11 and torch 2.14.0+cu132. - -Native safetensors weights only, not a Transformers AutoModel package or full training-resume bundle. Tensor names and mixed dtypes are preserved. Null Megatron extra-state entries are recorded explicitly in config.json. Optimizer/scheduler/RNG/dataloader state is excluded. The JSON model arguments are the effective saved settings; they override the base launch snapshot. K3 denotes three feedback passes, each with two shared-stack depth loops and terminal refiners. - -## Native CPU loading - -```python -import json, torch, importlib, tempfile -from argparse import Namespace -from megatron.core import parallel_state -from megatron.core.process_groups_config import ProcessGroupCollection -from full_bandwidth_loop_tf.megatron.config import build_model_config -from full_bandwidth_loop_tf.megatron.builder import LoopedFullBandwidthMegatronBuilder -from safetensors.torch import load_file - -def decode(x): - if isinstance(x, dict): - if "__torch_dtype__" in x: return getattr(torch, x["__torch_dtype__"]) - if "__dataclass__" in x: - module, cls, values = x["__dataclass__"] - return getattr(importlib.import_module(module), cls)(**decode(values)) - if "__enum__" in x: - module, cls, member = x["__enum__"] - return getattr(importlib.import_module(module), cls)[member] - return {k: decode(v) for k, v in x.items()} - if isinstance(x, list): return [decode(v) for v in x] - return x - -c = json.load(open("config.json")) -a = Namespace(**decode(c["model_arguments"])) -a.use_cpu_initialization = True -a.perform_initialization = False -cfg = build_model_config(a) -# Standalone CPU loading; use your existing groups in an initialized runtime. -with tempfile.TemporaryDirectory() as tmp: - torch.distributed.init_process_group("gloo", init_method="file://" + tmp + "/rdzv", rank=0, world_size=1) - parallel_state.initialize_model_parallel() - with torch.device("meta"): - model = LoopedFullBandwidthMegatronBuilder(cfg).build_model(ProcessGroupCollection.use_mpu_process_groups()) - state = load_file("model.safetensors") | c["non_tensor_state"] - model.load_state_dict(state, strict=True, assign=True) - model.eval() - torch.distributed.destroy_process_group() -``` - -All exported tensor values/shapes/dtypes/names are exactly equal to the source, all finite, and strict native loading passed. manifest.json hashes every payload file except itself. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/config.json deleted file mode 100644 index e6719b5cae5f8bc9822ede03b4e16a5389a34f3b..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/config.json +++ /dev/null @@ -1,1379 +0,0 @@ -{ - "checkpoint_format": "safetensors_native_megatron_state_dict", - "architecture": "full_bandwidth_loop", - "backbone_profile": "paper2604_vanilla", - "run_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "iteration": 8000, - "checkpoint_version": 3.0, - "training_code_revision": "8ce34798f6137fdaa27e6dfccc9323b506142b83", - "training_base_revision": "69969725a91703e9999827ebcda2b2d9b7e9c941", - "source_provenance": "Training used a modified snapshot based on training_base_revision. The entire snapshot src tree exactly matches the later committed training_code_revision recorded here.", - "backbone_source_revision": "122366472a31e57982a2877eeee10bb115477713", - "parameter_counts": { - "total": 653383808, - "trainable": 653383808 - }, - "model_arguments": { - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_size": 1, - "sequence_parallel": false, - "context_parallel_size": 1, - "hierarchical_context_parallel_sizes": null, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "perform_initialization": true, - "use_cpu_initialization": null, - "fp16": false, - "bf16": true, - "deterministic_mode": false, - "tp_comm_overlap": false, - "tp_comm_bulk_wgrad": true, - "tp_comm_bulk_dgrad": true, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "cross_entropy_loss_fusion": false, - "cross_entropy_fusion_impl": "native", - "tp_comm_bootstrap_backend": "nccl", - "overlap_moe_expert_parallel_comm": false, - "delay_wgrad_compute": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "ep_overlap_early_attn_memory_release": false, - "use_ring_exchange_p2p": false, - "defer_embedding_wgrad_compute": false, - "wgrad_deferral_limit": 0, - "overlap_p2p_comm_warmup_flush": false, - "microbatch_group_size_per_vp_stage": null, - "mtp_standalone": false, - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "num_layers": 20, - "mtp_num_layers": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_use_repeated_layer": false, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "hidden_size": 1408, - "num_attention_heads": 11, - "attention_backend": { - "__enum__": [ - "megatron.core.transformer.enums", - "AttnBackend", - "auto" - ] - }, - "flash_attention_version": null, - "softmax_type": "vanilla", - "num_query_groups": 11, - "ffn_hidden_size": 5632, - "kv_channels": 128, - "hidden_dropout": 0.0, - "attention_dropout": 0.0, - "fp32_residual_connection": false, - "apply_residual_connection_post_layernorm": false, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "add_bias_linear": false, - "add_qkv_bias": false, - "glu_linear_offset": 0.0, - "activation_func_clamp_value": null, - "rotary_interleaved": false, - "normalization": "RMSNorm", - "qk_layernorm": false, - "qk_l2_norm": false, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "attention_output_gate": false, - "calculate_per_token_loss": false, - "multi_latent_attention": false, - "experimental_attention_variant": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_head_dim": null, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "csa_window_size": 128, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_value_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "init_method_std": 0.02, - "embedding_init_method_std": null, - "init_model_with_meta_device": false, - "use_mup": false, - "mup_width_mult": 1.0, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_base_head_dim": null, - "mup_attn_scale_power": 1.0, - "apply_query_key_layer_scaling": false, - "disable_bf16_reduced_precision_matmul": false, - "use_fused_weighted_squared_relu": false, - "fused_residual_rmsnorm": false, - "use_transformer_engine_op_fuser": false, - "recompute_granularity": null, - "recompute_method": null, - "recompute_num_layers": null, - "distribute_saved_activations": false, - "recompute_modules": null, - "fp8": null, - "fp8_recipe": "delayed", - "fp8_quantizer_factory": null, - "fp8_margin": 0, - "fp8_interval": 1, - "fp8_amax_history_len": 1, - "fp8_amax_compute_algo": "most_recent", - "fp8_wgrad": true, - "fp8_output_proj": false, - "first_last_layers_bf16": false, - "num_layers_at_start_in_bf16": 1, - "num_layers_at_end_in_bf16": 1, - "use_kitchen_attention": false, - "kitchen_attention_backend": "sdpa", - "fp4": null, - "fp4_recipe": "nvfp4", - "fp4_quantizer_factory": null, - "high_priority_a2a_comm_stream": false, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_overlap": false, - "use_grouped_gemm_for_shared_expert": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_ffn_hidden_size": null, - "moe_router_topk": 2, - "moe_enable_routing_replay": false, - "moe_router_padding_for_quantization": false, - "moe_router_padding_for_fp8": false, - "moe_router_num_groups": null, - "moe_router_group_topk": null, - "moe_router_pre_softmax": false, - "moe_router_topk_scaling_factor": null, - "moe_router_score_function": "softmax", - "moe_router_dtype": null, - "moe_router_skip_muon": false, - "moe_router_enable_expert_bias": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_force_load_balancing": false, - "moe_router_force_biased": null, - "use_grouped_gemm_for_dense_mlp": false, - "moe_grouped_gemm": false, - "moe_use_grouped_tensor": false, - "moe_single_grouped_weight": false, - "moe_single_grouped_bias": false, - "moe_z_loss_coeff": null, - "moe_input_jitter_eps": null, - "moe_token_dispatcher_type": "allgather", - "moe_enable_deepep": false, - "moe_flex_dispatcher_backend": "deepep", - "moe_permute_fusion_into_hybridep": false, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_per_layer_logging": false, - "moe_expert_capacity_factor": null, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_token_drop_policy": "probs", - "moe_layer_recompute": false, - "moe_permute_fusion": false, - "moe_router_fusion": false, - "moe_apply_probs_on_input": false, - "moe_latent_size": null, - "moe_use_norm_before_up_proj": false, - "gtp_remat_opt_in_modules": [], - "moe_flex_dispatcher_num_sms": null, - "moe_deepep_num_sms": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_ncclep_zero_copy": false, - "moe_dispatch_fwd_dtype": "bf16", - "moe_combine_bwd_dtype": "bf16", - "moe_mlp_glu_interleave_size": null, - "moe_expert_rank_capacity_factor": null, - "linear_cp_mode": "headwise", - "linear_cp_layout": "zigzag", - "attention_cp_layout": "zigzag", - "enable_cuda_graph": false, - "cuda_graph_warmup_steps": 3, - "external_cuda_graph": false, - "cuda_graph_impl": "none", - "inference_cuda_graph_scope": { - "__enum__": [ - "megatron.core.transformer.enums", - "InferenceCudaGraphScope", - "none" - ] - }, - "enable_mhc_connections": false, - "mhc_num_residual_streams": 4, - "mhc_sinkhorn_iterations": 20, - "mhc_init_gating_factor": 0.01, - "use_fused_mhc": false, - "mhc_fused_backend": "auto", - "mhc_recompute_layer_num": null, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "flash_decode": false, - "batch_invariant_mode": false, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "use_te_activation_func": false, - "symmetric_ar_type": null, - "nccl_all_reduce_for_prefill": false, - "inference_fuse_tp_communication": false, - "inference_disable_triton_nvls_kernels": false, - "inference_grouped_gemm_backend": "vllm", - "inference_moe_disable_fused_quant_kernels": false, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_moe_token_dispatcher_type": "nvls", - "mrope_section": null, - "is_hybrid_model": false, - "mamba_state_dim": 128, - "mamba_head_dim": 64, - "mamba_num_groups": 8, - "mamba_num_heads": null, - "gdp_num_householder": 3, - "use_mamba_mem_eff_path": true, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "transformer_impl": "local", - "fine_grained_activation_offloading": false, - "offload_modules": [], - "min_offloaded_tensor_size": 1048576, - "delay_offload_until_cuda_graph": false, - "delta_offload_bytes_across_pp_ranks": 0, - "activation_offload_fraction": 1.0, - "moe_paged_stash": false, - "moe_paged_stash_page_size": 64, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "fine_grained_offloading_max_inflight_offloads": null, - "encoder_num_layers": 20, - "decoder_num_layers": null, - "group_query_attention": false, - "window_size": null, - "window_attn_skip_freq": null, - "max_position_embeddings": 2048, - "position_embedding_type": "rope", - "yarn_original_max_position_embeddings": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "relative_attention_num_buckets": 32, - "relative_attention_max_distance": 128, - "use_rotary_position_embeddings": false, - "rotary_base": 10000, - "rotary_percent": 1.0, - "rotary_seq_len_interpolation_factor": null, - "use_rope_scaling": false, - "rope_scaling_factor": 8.0, - "no_rope_freq": null, - "add_position_embedding": true, - "make_vocab_size_divisible_by": 64, - "openai_gelu": false, - "squared_relu": false, - "swiglu": false, - "quick_geglu": false, - "onnx_safe": null, - "bert_binary_head": true, - "untie_embeddings_and_output_weights": true, - "weight_decay": 0.0, - "apply_wd_to_qk_layernorm": false, - "clip_grad": 0.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "sgd_momentum": 0.9, - "muon_momentum": 0.95, - "muon_split_qkv": true, - "muon_nesterov": true, - "muon_scale_mode": "spectral", - "muon_fp32_matmul_prec": "medium", - "muon_coefficient_type": "quintic", - "muon_num_ns_steps": 5, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "muon_extra_scale_factor": 1.0, - "muon_scalar_optimizer": "adam", - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "no_weight_decay_cond_type": null, - "profile": false, - "profile_step_start": 10, - "profile_step_end": 12, - "use_pytorch_profiler": false, - "pytorch_profiler_collect_shapes": false, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "profile_ranks": [], - "record_memory_history": false, - "memory_snapshot_path": "snapshot.pickle", - "record_shapes": false, - "nvtx_ranges": false, - "micro_batch_size": 4, - "global_batch_size": 128, - "rampup_batch_size": null, - "step_batch_size_schedule": null, - "decrease_batch_size_if_needed": false, - "empty_unused_memory_level": 0, - "check_weight_hash_across_dp_replicas_interval": null, - "gpu_sniff_test_interval": null, - "train_sync_interval": null, - "train_iters": 26703, - "train_samples": null, - "exit_interval": null, - "exit_duration_in_mins": null, - "exit_signal_handler": false, - "exit_signal": { - "__enum__": [ - "signal", - "Signals", - "SIGTERM" - ] - }, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "manual_gc": false, - "manual_gc_interval": 0, - "manual_gc_eval": true, - "iterations_to_skip": [], - "check_for_nan_in_loss_and_grad": true, - "check_for_large_grads": false, - "result_rejected_tracker_filename": null, - "tp_comm_overlap_cfg": null, - "masked_softmax_fusion": true, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "bias_dropout_fusion": false, - "apply_rope_fusion": false, - "rope_type": null, - "use_flash_attn": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "use_torch_optimizer_for_cpu_offload": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "dump_param_to_param_group_map": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "dataloader_type": "external", - "no_persist_layer_norm": false, - "gradient_accumulation_fusion": false, - "deprecated_use_mcore_models": false, - "perform_rl_step": false, - "rl_prompts_per_eval": 32, - "grpo_prompts_per_step": 32, - "grpo_group_size": 2, - "rl_generation_lag": null, - "rl_max_inflight_requests": null, - "rl_submission_granularity": "B", - "rl_consumption_granularity": "B", - "rl_durable_rollout_bank": false, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "grpo_iterations": 2, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_kl_beta": 0.001, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "langrl_env_config": null, - "rl_default_temperature": 1.0, - "rl_default_top_p": 0, - "rl_default_top_k": -1, - "rl_offload_optimizer_during_inference": false, - "rl_kv_cache_management_mode": "persist", - "rl_persist_cuda_graphs": false, - "rl_partial_rollouts": false, - "rl_inference_logprobs_is_correction": false, - "rl_importance_sampling_truncation_coef": null, - "rl_use_sequence_packing": false, - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_sequence_packing_algo": "fifo", - "rl_training_cuda_graphs": false, - "rl_inference_tensor_model_parallel_size": null, - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_model_unified_memory_level": 0, - "rl_offload_inference_model_weights_when_idle": false, - "refit_method": "gloo", - "refit_execution_batch_bytes": null, - "rl_verify_model_weights_swap": false, - "rl_skip_bos_token": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_inference_parsers": [], - "seed": 42, - "te_rng_tracker": false, - "inference_rng_tracker": false, - "data_parallel_random_init": false, - "init_method_xavier_uniform": false, - "lr_decay_style": "linear", - "lr_wsd_decay_style": "linear", - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_warmup_fraction": null, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_warmup_init": 0.0, - "override_opt_param_scheduler": false, - "use_checkpoint_opt_param_scheduler": false, - "start_weight_decay": 0.0, - "end_weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "lr": 0.007470638533717786, - "min_lr": 0.0007470638533717787, - "decoupled_lr": null, - "decoupled_min_lr": null, - "freeze_all_layers": false, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/checkpoints", - "save_interval": 1000, - "save_params_interval": null, - "save_activations_interval": null, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "save_dgrads_interval": null, - "save_retain_interval": null, - "load": null, - "load_main_params_from_ckpt": false, - "non_persistent_save_interval": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "finetune": false, - "pretrained_checkpoint": null, - "ckpt_step": null, - "use_checkpoint_args": false, - "use_mp_args_from_checkpoint_args": false, - "use_tokenizer_model_from_checkpoint_args": true, - "exit_on_missing_checkpoint": false, - "ckpt_format": "torch", - "auto_detect_ckpt_format": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_fully_parallel_save": true, - "async_save": false, - "async_strategy": "mcore", - "use_persistent_ckpt_worker": false, - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_assume_constant_structure": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_pg_tensors_cache_create": false, - "ckpt_load_validate_sharding_integrity": true, - "strict_fsdp_dtensor_load": true, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_optim_fully_reshardable": false, - "distrib_optim_fully_reshardable_mem_efficient": false, - "save_tokenizer_assets": true, - "replication": false, - "replication_jump": null, - "replication_factor": 2, - "verify_integrity": false, - "no_save_optim": null, - "no_save_rng": null, - "no_load_optim": null, - "no_load_rng": null, - "override_ckpt_iteration": null, - "use_dist_ckpt_deprecated": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_workers": 1, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_drop_redundant_extra_state": false, - "grad_reduce_in_bf16": false, - "loss_scale": null, - "initial_loss_scale": 4294967296, - "min_loss_scale": 1.0, - "loss_scale_window": 1000, - "hysteresis": 2, - "attention_softmax_in_fp32": false, - "accumulate_allreduce_grads_in_fp32": true, - "fp16_lm_cross_entropy": false, - "logit_dtype": null, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "mamba_training_ssm_states_dtype": null, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "align_grad_reduce": true, - "local_rank": 0, - "lazy_mpu_init": false, - "use_megatron_fsdp": false, - "use_torch_fsdp2": false, - "nccl_communicator_config_path": null, - "use_tp_pp_dp_mapping": false, - "use_gloo_process_groups": true, - "use_sharp": false, - "sharp_enabled_group": null, - "high_priority_stream_groups": [], - "distributed_timeout_seconds_after_init": null, - "flight_recorder_dump_path": null, - "flight_recorder_trace_buffer_size": 2000, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_include_only_active": true, - "flight_recorder_extra_dump_on_exec": true, - "disable_jit_fuser": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "pipeline_model_parallel_layout": null, - "num_layers_per_virtual_pipeline_stage": null, - "num_virtual_stages_per_pipeline_rank": null, - "overlap_p2p_comm": false, - "overlap_grad_reduce": false, - "ddp_num_buckets": null, - "ddp_bucket_size": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_reduce_scatter_with_fp32_accumulation": false, - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": [], - "ddp_average_in_collective": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "align_param_gather": false, - "use_distributed_optimizer": false, - "megatron_fsdp_version": 1, - "use_layer_wise_param_layout": true, - "nccl_ub": false, - "disable_symmetric_registration": false, - "gtp_remat_nccl_ub": false, - "gtp_expert_remat_nccl_ub": false, - "fsdp_manual_registration": false, - "create_all_gather_group": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "expert_data_parallel_sharding_strategy": null, - "outer_dp_sharding_strategy": "no_shard", - "hfsdp_param_gather_overlap": false, - "gradient_reduce_div_fusion": true, - "fsdp_double_buffer": false, - "suggested_communication_unit_size": null, - "keep_fp8_transpose_cache": false, - "enable_full_sharding_in_hsdp": false, - "num_distributed_optimizer_instances": 1, - "torch_fsdp2_reshard_after_forward": true, - "cp_comm_type": [ - "p2p" - ], - "max_seqlen_per_dp_cp_rank": null, - "hybrid_context_parallel": false, - "sequence_packing_scheduler": null, - "fake_process_group": false, - "eval_iters": 2, - "eval_interval": 100, - "start_eval_at_iter": null, - "eval_global_batch_size": 128, - "eval_micro_batch_size": 4, - "skip_train": false, - "test_mode": false, - "full_validation": false, - "multiple_validation_sets": false, - "validation_set_names": null, - "data_path": null, - "phase_transition_iterations": null, - "split": null, - "train_data_path": null, - "valid_data_path": null, - "test_data_path": null, - "data_args_path": null, - "per_split_data_args_path": null, - "per_dataset_sequences_path": null, - "dataloader_fast_cache_load": false, - "dataloader_defer_npy_index_mmap": false, - "data_cache_path": null, - "mmap_bin_files": true, - "mock_data": false, - "seq_length": 2048, - "encoder_seq_length": 2048, - "decoder_seq_length": null, - "sample_rate": 1.0, - "mask_prob": 0.15, - "short_seq_prob": 0.1, - "num_workers": 0, - "reset_position_ids": false, - "reset_attention_mask": false, - "dataloader_inter_document_masking": false, - "eod_mask_loss": false, - "create_attention_mask_in_dataloader": false, - "num_dataset_builder_threads": 1, - "object_storage_cache_path": null, - "mid_level_dataset_surplus": 0.005, - "allow_ambiguous_pad_tokens": false, - "fim_data": false, - "fim_rate": 0.5, - "fim_spm_rate": 0.5, - "fim_split_sample": null, - "fim_fragment_rate": null, - "fim_no_prefix": null, - "fim_prefix_token": "", - "fim_middle_token": "", - "fim_suffix_token": "", - "fim_pad_token": "", - "fim_eod_token": "<|endoftext|>", - "vocab_size": 32001, - "padded_vocab_size": 32064, - "pad_vocab_size": true, - "vocab_file": null, - "merge_file": null, - "vocab_extra_ids": 0, - "tokenizer_type": "HuggingFaceTokenizer", - "tokenizer_model": "/taijifs_wzba/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "metadata_path": null, - "special_tokens": null, - "tiktoken_pattern": null, - "tiktoken_num_special_tokens": 1000, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_hf_no_use_fast": false, - "tokenizer_hf_no_include_special_tokens": false, - "trust_remote_code": false, - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "chat_template": null, - "use_gigatoken": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "ict_head_size": null, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "ict_load": null, - "bert_load": null, - "titles_data_path": null, - "query_in_block_prob": 0.1, - "use_one_sent_docs": false, - "evidence_data_path": null, - "retriever_report_topk_accuracies": [], - "retriever_score_scaling": false, - "block_data_path": null, - "embedding_path": null, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "num_classes": 1000, - "img_h": 224, - "img_w": 224, - "num_channels": 3, - "patch_dim": 16, - "classes_fraction": 1.0, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "head_lr_mult": 1.0, - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vision_backbone_type": "vit", - "swin_backbone_type": "tiny", - "mask_type": "random", - "mask_factor": 1.0, - "iter_per_epoch": 1250, - "dino_local_img_size": 96, - "dino_local_crops_number": 10, - "dino_head_hidden_size": 2048, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_norm_last_layer": false, - "dino_warmup_teacher_temp": 0.04, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp_epochs": 30, - "num_experts": null, - "moe_layer_freq": 1, - "moe_use_upcycling": false, - "moe_router_load_balancing_type": "aux_loss", - "moe_aux_loss_coeff": 0.0, - "moe_upcycling_granularity": 1, - "q_lora_rank": null, - "kv_lora_rank": 32, - "qk_head_dim": 128, - "qk_pos_emb_head_dim": 64, - "v_head_dim": 128, - "rotary_scaling_factor": 1.0, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "cache_mla_latents": false, - "mla_down_proj_fusion": false, - "linear_attention_freq": null, - "heterogeneous_layers_config_path": null, - "heterogeneous_layers_config_encoded_json": null, - "log_interval": 1, - "log_params_norm": false, - "log_throughput": false, - "log_progress": false, - "timing_log_level": 0, - "timing_log_option": "minmax", - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "log_timers_to_tensorboard": true, - "log_loss_scale_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_memory_to_tensorboard": false, - "log_memory_interval": null, - "log_device_memory_used": false, - "log_num_zeros_in_grad": false, - "log_max_attention_logit": false, - "barrier_with_L1_time": true, - "log_world_size_to_tensorboard": false, - "wandb_project": "loopTF-workspace", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/wandb", - "wandb_entity": "LoopTF-4-CSPs", - "logging_level": null, - "log_energy": false, - "moe_routing_trace_path": null, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_dump_weights": false, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_load_dir": null, - "logits_load_decode_threads": 4, - "logits_load_prefetch_factor": 3, - "logits_load_msc_prefetch_depth": 2, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_ignore_errors": false, - "log_straggler": false, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "disable_straggler_on_startup": false, - "run_workload_inspector_server": false, - "inference_batch_times_seqlen_threshold": -1, - "max_tokens_to_oom": 12000, - "output_bert_embeddings": false, - "bert_embedder_type": "megatron", - "cuda_graph_modules": [], - "use_legacy_static_engine": false, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_dynamic_batching": false, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_track_generated_token_events": false, - "decode_only_cuda_graphs": false, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_dynamic_batching_unified_memory_level": 0, - "enable_chunked_prefill": false, - "num_speculative_tokens": 0, - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_sampling_backend": "torch", - "offset_sampling_seed_by_dp_rank": true, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_logging_step_interval": 0, - "inference_text_gen_server_logging": false, - "inference_wandb_logging": false, - "inference_coordinator_port": null, - "mamba_inference_conv_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "mamba_inference_ssm_states_dtype": { - "__torch_dtype__": "bfloat16" - }, - "inference_use_synchronous_zmq_collectives": false, - "inference_disable_ep_consensus": false, - "inference_shards": null, - "fp4_param_gather": false, - "fp8_param_gather": false, - "te_precision_config_file": null, - "enable_experimental": false, - "spec": null, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "yaml_cfg": null, - "use_precision_aware_optimizer": false, - "main_grads_dtype": { - "__torch_dtype__": "float32" - }, - "main_params_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_dtype": { - "__torch_dtype__": "float32" - }, - "exp_avg_sq_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_params_dtype": { - "__torch_dtype__": "float32" - }, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_max_pool_double_buffer": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "enable_one_logger": true, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "one_logger_async": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "inprocess_restart": false, - "inprocess_max_iterations": null, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_monitor_process_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_heartbeat_interval": 30, - "inprocess_soft_timeout": 60, - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_timeout": 60, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_last_call_wait": 1, - "inprocess_termination_grace_time": 1, - "inprocess_granularity": "node", - "inprocess_active_world_size": 16, - "inprocess_empty_cuda_cache": false, - "enable_ft_package": false, - "calc_ft_timeouts": false, - "ft_num_warmup_iters": 5, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "rerun_mode": "validate_results", - "check_for_spiky_loss": false, - "enable_msc": false, - "disable_msc_deprecated": false, - "sft": false, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sft_mock_dataset_config_json": null, - "use_varlen_dataset": false, - "varlen_sbhd_validation": false, - "varlen_mock_dataset_config_json": null, - "fault_injector_ranks": null, - "fault_injector_num_ranks": null, - "fault_injector_fault_types": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_delay": null, - "fault_injector_delay_start_iteration": null, - "fault_injector_mtti_seconds": null, - "fault_injector_offset_seconds": null, - "fault_injector_seed": null, - "architecture": "full_bandwidth_loop", - "architecture_module": [], - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "training_recipe": "paper2604_21106", - "recipe_overrides": {}, - "recipe_requested_batch_tokens": 262144, - "paper_reference_width": 640, - "paper_reference_tokens": 983315200, - "paper_reference_batch_tokens": 262144, - "paper_embedding_lr": 0.3, - "paper_unembedding_lr": 0.004, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "online_data": true, - "online_pack_sequences": false, - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_subset": "sample-100BT", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_source_split": "train", - "online_text_column": "text", - "online_validation_fraction": 0.001, - "online_split_seed": 260421106, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_tokenizer_batch_size": 256, - "online_shuffle_buffer_records": 128, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_prefetch_batches": 16, - "online_source_mode": "local", - "online_raw_manifest": "/root/dl/runtime/script/loopTF-workspace/data/raw/fineweb-edu-100BT/manifest.json", - "online_validation_manifest": null, - "online_latency_log_dir": null, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/20260917-234527_loop-refiner-norm-product/dataloader", - "eval_at_start": true, - "final_eval_iters": 270, - "backbone_profile": "paper2604_vanilla", - "aux_mtp_horizon": 1, - "aux_mtp_rank": 128, - "aux_mtp_loss_weight": 0.3, - "feedback_loss_weight": 1.0, - "feedback_jitter_sigma": 0.0, - "feedback_prefix_mixin": false, - "feedback_pass_mixture": "3:1.0", - "feedback_progressive_stages": "[]", - "feedback_eval_passes": 3, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "feedback_log_hidden_change": false, - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_normuon_beta2": 0.95, - "depth_num_loops": 2, - "loop_final_adapter": "lookahead_refiner", - "loop_adapter_ffn_size": 5632, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "adaptive_muon_moment2_method": "normuon", - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "_is_global_batch_size_explicitly_specified": true, - "rank": 0, - "world_size": 16, - "use_dist_ckpt": false, - "gtp_weight_remat_size": 1, - "transformer_pipeline_model_parallel_size": 1, - "data_parallel_size": 16, - "virtual_pipeline_model_parallel_size": null, - "params_dtype": { - "__torch_dtype__": "bfloat16" - }, - "consumed_train_samples": 1024000, - "skipped_train_samples": 0, - "consumed_valid_samples": 20736, - "variable_seq_lengths": false, - "expert_gtp_weight_remat_size": 1, - "use_layer_wise_distributed_optimizer": false, - "recipe_optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "recipe_schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "paper_base_matrix_lr": 0.014, - "paper_transfer": { - "__dataclass__": [ - "loop_transformer_backends.recipes", - "Paper2604Transfer", - { - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144 - } - ] - }, - "training_recipe_metadata": { - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } - }, - "backbone_metadata": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - }, - "evaluation_metadata": { - "eval_at_start": true, - "initial_iters": 2, - "intermediate_iters": 2, - "final_validation_iters": 270, - "test_iters": 0, - "global_batch_size": 128, - "sequence_length": 2048, - "intermediate_input_tokens": 524288, - "final_validation_sequences": 34560, - "final_validation_input_tokens": 70778880, - "validation_iterator": "continues_existing_held_out_iterator" - }, - "iteration": 0, - "num_floating_point_operations_so_far": 0, - "do_train": 1, - "do_valid": 1, - "do_test": 0, - "curr_iteration": 7999 - }, - "non_tensor_state": { - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "output_layer._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null - }, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "training_tokens": 2097152000, - "training_budget_tokens": 7000031232, - "tokenizer": { - "repository": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "added_tokens": [ - "<|assistant_end|>" - ], - "active_vocab_size": 32001, - "padded_vocab_size": 32064 - }, - "loading": "Construct matching native Megatron model from model_arguments, restore non_tensor_state, load_state_dict(strict=True, assign=True). See README." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/launch_config.env deleted file mode 100644 index fd20343c3156687768652281c7bb3e49c6a29035..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/launch_config.env +++ /dev/null @@ -1,111 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Ordinary shared-stack loops; first-loop KV banks feed the terminal adapter. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/manifest.json deleted file mode 100644 index 4c336f691acb11cc225fa51ad9dc32a54abb1b3b..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "size": 3194, - "sha256": "fc90ea8863dfc989d255649387e77f530fc22b1ed91021b2121a6755a81e2e64" - }, - "config.json": { - "size": 52753, - "sha256": "1d7ef50bbc9ee61d3c7c511bbf6d8f4b0f8883127de211c4e44b1b30d0a175dc" - }, - "launch_config.env": { - "size": 3977, - "sha256": "b6f04e1a4c5d567c4ce7bbb404b684a68aa39ab2f19944ee952707d321ccf99d" - }, - "model.safetensors": { - "size": 1306923584, - "sha256": "17fabadec4c73bbfd8a245d19ebae7e68a13af41cb1f3b76e6e3207be77fd2cb" - }, - "training_recipe.json": { - "size": 4631, - "sha256": "959b18544fc1a69932223adb5b170afb938bfaa681bf4495163a5aca7b4be14c" - } - }, - "note": "Manifest does not hash itself." -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/model.safetensors deleted file mode 100644 index bc42c3b4e7289daca7d1b2f13f4438089462311f..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17fabadec4c73bbfd8a245d19ebae7e68a13af41cb1f3b76e6e3207be77fd2cb -size 1306923584 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/training_recipe.json deleted file mode 100644 index 33c9ed6cd74ea823b8c5aa2a6b77a9822a22b0d0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm/step_8000/training_recipe.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/README.md deleted file mode 100644 index 98b9c3ea7e3df111eea9cd5ab55d4ed577c0633c..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 12000 - -Intermediate checkpoint at step **12,000**, after **3,145,728,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_12000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/config.json deleted file mode 100644 index 74a8f5a636b90eb178a25cb97ed8a4d6967e431a..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 12000, - "checkpoint_version": 3.0, - "consumed_train_samples": 1536000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 4000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 12000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 3145728000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/manifest.json deleted file mode 100644 index 77dba4e2193a787fe25b82dd518017425c53b697..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 12000, - "files": { - "README.md": { - "sha256": "4dceff7f6bfba14b3354db91c527304fe949a08d7c569ae58ee9c57aab31e7fc", - "size": 2805 - }, - "config.json": { - "sha256": "b926c0613179b1aa07d80de5190bc26a779365a8a20f57009d4456604c883872", - "size": 13664 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "03142f09e76eab669b139820512f9e56e6376201079d18f9da81b4b9ea51e967", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/model.safetensors deleted file mode 100644 index 2c40b08a401dae5d61aca9f56d9c82c99a2034c4..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:03142f09e76eab669b139820512f9e56e6376201079d18f9da81b4b9ea51e967 -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_12000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/README.md deleted file mode 100644 index f2d3ee7c57c852382176644ffb4b01b8e727a5e3..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 16000 - -Intermediate checkpoint at step **16,000**, after **4,194,304,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_16000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/config.json deleted file mode 100644 index 64557094996148145bd09ea865b7a675ddf3fc71..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 16000, - "checkpoint_version": 3.0, - "consumed_train_samples": 2048000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 4000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 16000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 4194304000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/manifest.json deleted file mode 100644 index f0c4a5dab0ac244a4398041e4b53ab601dfe0001..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 16000, - "files": { - "README.md": { - "sha256": "f773b3be00cb468e8d85df62f958576c5dbcf30b1634eaf7b908c4b2a8087a48", - "size": 2805 - }, - "config.json": { - "sha256": "b6f02ed5e868c37dba878ec55a586854a2488de3189660f08546f695b03ecb04", - "size": 13664 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "f57db373c869fc8a513542b6d885981b9a98342239a1448162b19019afab738a", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/model.safetensors deleted file mode 100644 index 309c89e311a07124890927849b8f82b4121a4878..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f57db373c869fc8a513542b6d885981b9a98342239a1448162b19019afab738a -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_16000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/README.md deleted file mode 100644 index 627d13ac01e683e417a063c1ebacc871afe54bdb..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 20000 - -Intermediate checkpoint at step **20,000**, after **5,242,880,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_20000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/config.json deleted file mode 100644 index 7d287cd8f081d7fc210566ec747c8b6dfe4259fa..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 20000, - "checkpoint_version": 3.0, - "consumed_train_samples": 2560000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 4000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 20000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 5242880000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/manifest.json deleted file mode 100644 index 419667e55c3e017468cbc13c66bc89f3937bad0d..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 20000, - "files": { - "README.md": { - "sha256": "d4e7558b1bc8a95ac7d61e1dfd8d6fce87b9ebb79b6e31aa5998d095a662468e", - "size": 2805 - }, - "config.json": { - "sha256": "31402b87ce67627b43551fc1d483b0ead99bcf3eea50a582e6a7b34491c5cfad", - "size": 13664 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "c7a2b9c3edff1f0d06de9c4de3ccce7f323af3aa8af52cb914679d253d2299d8", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/model.safetensors deleted file mode 100644 index 1f29bcb43b93e19819ee4e64c50902425ee8ea64..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7a2b9c3edff1f0d06de9c4de3ccce7f323af3aa8af52cb914679d253d2299d8 -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_20000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/README.md deleted file mode 100644 index f79fcc2c22c713e248bab93a20b65a706e0e29b6..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 24000 - -Intermediate checkpoint at step **24,000**, after **6,291,456,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_24000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/config.json deleted file mode 100644 index 57e44090baba6f39f87101ca59a2252c023d0a0a..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 24000, - "checkpoint_version": 3.0, - "consumed_train_samples": 3072000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 4000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 24000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 6291456000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/manifest.json deleted file mode 100644 index 6bdb0163bb10996042525b77f9c3482546041073..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 24000, - "files": { - "README.md": { - "sha256": "5fce03d28b96c543ba4b87c823f6c474e4997c21dbe2293d69ea1dbcd26aec1f", - "size": 2805 - }, - "config.json": { - "sha256": "403fd0e82772d760eae288ed76bdb0a16a4f2398e5a305234e4e80102760d88e", - "size": 13664 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "4364bf9aaf8fc70f68aff8118d8238f724619f173f4129aaa3a982395ba0ec7b", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/model.safetensors deleted file mode 100644 index 8583536df993c47f1e79344dcda56113a3202a57..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4364bf9aaf8fc70f68aff8118d8238f724619f173f4129aaa3a982395ba0ec7b -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_24000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/README.md deleted file mode 100644 index d24a7720eb59b505c4ae7386ab30cebee9e07006..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 26703 - -Final saved checkpoint after **7,000,031,232 training input tokens**. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_26703") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -Final validation: pass-3 CE 2.281249761581421, perplexity 9.78890656739842, 34,560 sequences / 70,778,880 input tokens; copied from the completed run receipt. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/config.json deleted file mode 100644 index 50db19c24ae419f0a9d153305f29af4f5fb6d729..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/config.json +++ /dev/null @@ -1,318 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 26703, - "checkpoint_version": 3.0, - "consumed_train_samples": 3417984, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": "2", - "DEPTH_SHARE_KV": "true", - "EXIT_INTERVAL": "", - "FEEDBACK_EVAL_PASSES": "3", - "FEEDBACK_JITTER_SIGMA": "0", - "FEEDBACK_LOG_HIDDEN_CHANGE": "false", - "FEEDBACK_LOSS_WEIGHT": "1.0", - "FEEDBACK_NORMUON_BETA2": "0.95", - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": "false", - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": "0.0005", - "FEEDBACK_SCALAR_WEIGHT_DECAY": "0.0", - "FEEDBACK_Z_LOSS_COEFFICIENT": "0.0", - "FEEDBACK_Z_LOSS_START_FRACTION": "1.0", - "GLOBAL_BATCH_SIZE": "128", - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": "true", - "LOOP_ADAPTER_FFN_SIZE": "5632", - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": "4", - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SEED": "42", - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": "26703", - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": { - "eval/checkpoint_step": 26703, - "eval/final_val/dataset_id": "config-a71f5512f13ffd6f", - "eval/final_val/final_ce": 2.281249761581421, - "eval/final_val/final_ppl": 9.78890656739842, - "eval/final_val/num_input_tokens": 70778880, - "eval/final_val/num_sequences": 34560, - "eval/final_val/num_tokens": 70778880, - "eval/final_val/pass_1_ce": 2.3062374591827393, - "eval/final_val/pass_1_ppl": 10.036590442057946, - "eval/final_val/pass_2_ce": 2.282487154006958, - "eval/final_val/pass_2_ppl": 9.801026783424485, - "eval/final_val/pass_3_ce": 2.281249761581421, - "eval/final_val/pass_3_ppl": 9.78890656739842, - "eval/final_val/pass_count": 3, - "eval/is_final": true - }, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 26703 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 7000031232, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/manifest.json deleted file mode 100644 index aa825ae5aad0bded2692109fb6854f9188c6a59e..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 26703, - "files": { - "README.md": { - "sha256": "563a72cf819681584761dff25be279416df8e9463fc7c05c31b44ecebfadd839", - "size": 2787 - }, - "config.json": { - "sha256": "def835b63b7f06f42e818a7fb3d5fae5cc9669986179525b0964da44fd0429e7", - "size": 14144 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "18bfe61598be7be58617695b4c3138554b27d0f9fbae9f10ff8080c928d5616a", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/model.safetensors deleted file mode 100644 index b15e94b7be91757315837d06a6ff6764f335cca8..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:18bfe61598be7be58617695b4c3138554b27d0f9fbae9f10ff8080c928d5616a -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_26703/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/README.md deleted file mode 100644 index af3c7432aa1319476dec7164bd57aac4db77195c..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 4000 - -Intermediate checkpoint at step **4,000**, after **1,048,576,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_4000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/config.json deleted file mode 100644 index 368f8367a6cea0fb27610be58f9a18755ee1de99..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 4000, - "checkpoint_version": 3.0, - "consumed_train_samples": 512000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 1000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 4000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 1048576000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/manifest.json deleted file mode 100644 index 47202d0af0dd1aa53a4046fecef51305c290f7c5..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 4000, - "files": { - "README.md": { - "sha256": "4962079217f16388a4a6fb7cfee7293d4e59a458265eab7cf3d397a7e5f4cff2", - "size": 2802 - }, - "config.json": { - "sha256": "2930c9ff7ad3ab2656fc9c4681cda446dc514a16fcf41720d993dd418f8b60e6", - "size": 13661 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "80dedbdc81729af94e0e7ad7a86009ceadf75c2c5f3cbdb50d4cdd133660ef05", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/model.safetensors deleted file mode 100644 index ded8edd42eab319fd182c86903492cfbb7b03030..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:80dedbdc81729af94e0e7ad7a86009ceadf75c2c5f3cbdb50d4cdd133660ef05 -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_4000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/README.md b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/README.md deleted file mode 100644 index 1d58031b7781877bbeabc74171fa92f61a9c4619..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531 — step 8000 - -Intermediate checkpoint at step **8,000**, after **2,097,152,000 training input tokens**, from a run with a 7,000,031,232-token total budget. Complete model: **653,383,808 parameters**. The `476m` label refers to the backbone sizing convention, and `7b` to the training-token budget. - -Native state-dict safetensors, preserving every source tensor name, shape, dtype and value. This is weights and configuration only, not a full training-resume bundle or a Transformers `AutoModel.from_pretrained` export. All source tensors were finite and strict native loading passed on CPU. Source checkpoints were retained. - -Training revision: `f859c42e740ac6179c0ab383a0e5334cbf75b3f2`; paper/backbone source revision: `122366472a31e57982a2877eeee10bb115477713`. Use [loopTF-workspace loading revision](https://github.com/FFishy-git/loopTF-workspace/tree/3763dea5b6c0cb4496405eff8d7a83f103193769) with `src` on `PYTHONPATH` and its dependencies installed. - -The original training checkout was dirty; `training_code.patch` preserves its recorded diff. The verified loading revision provides the existing CPU inference adapter for the same shared-KV, normalized terminal-refiner architecture. This adapter maps packed QKV/key names at load time and uses full-prefix inference; the uploaded tensors keep native Megatron names. - -## Loading - -```python -import json -from pathlib import Path -import torch -from safetensors.torch import load_file -export_dir = Path("/path/to/step_8000") -config = json.loads((export_dir / "config.json").read_text()) -state = load_file(str(export_dir / "model.safetensors"), device="cpu") -from types import SimpleNamespace -from loop_transformer_backends.benchmarks.models import build_native -model, metadata = build_native({ - "args": SimpleNamespace(**config["native_loading_args"]), - "model": state | config["non_tensor_state_dict_entries"], - "iteration": config["checkpoint_iteration"], -}) # Existing project inference mapping, strict=True, assign=True internally. -model.eval() -``` - -Use the pinned Llama-2 tokenizer plus `<|assistant_end|>` in `config.json`, with 32,001 active tokens and weights padded to 32,064. `launch_config.env` is the original reviewed base snapshot; its relative imports require the pinned source tree. Effective timestamped run identity and overrides are recorded in `config.json`, which is authoritative for loading. - -This is an intermediate training checkpoint. No final-evaluation result is attributed to these weights. - -W&B: [source run](https://wandb.ai/LoopTF-4-CSPs/loopTF-workspace/runs/n8n2vo7q). `manifest.json` records file sizes and SHA256 and does not hash itself. No optimizer, scheduler, RNG, dataloader, dataset, logs or credentials are included. diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/config.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/config.json deleted file mode 100644 index 49237cd8a8f586cb6a73c8bfd8f178a61e9f6ba8..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "backbone_profile": "paper2604_vanilla", - "bias": false, - "block_application_multiplier": 6.449999999999999, - "block_application_multiplier_per_pass": 2.15, - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_iteration": 8000, - "checkpoint_version": 3.0, - "consumed_train_samples": 1024000, - "crop_padding_before_loss": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "depth_num_loops": 2, - "depth_share_kv": true, - "dropout": 0.0, - "effective_overrides": { - "DEPTH_NUM_LOOPS": 2, - "DEPTH_SHARE_KV": true, - "EXIT_INTERVAL": null, - "FEEDBACK_EVAL_PASSES": 3, - "FEEDBACK_JITTER_SIGMA": 0.0, - "FEEDBACK_LOG_HIDDEN_CHANGE": false, - "FEEDBACK_LOSS_WEIGHT": 1.0, - "FEEDBACK_NORMUON_BETA2": 0.95, - "FEEDBACK_PASS_MIXTURE": "3:1.0", - "FEEDBACK_PREFIX_MIXIN": false, - "FEEDBACK_PROGRESSIVE_STAGES": "[]", - "FEEDBACK_SCALAR_LR": 0.0005, - "FEEDBACK_SCALAR_WEIGHT_DECAY": 0.0, - "FEEDBACK_Z_LOSS_COEFFICIENT": 0.0, - "FEEDBACK_Z_LOSS_START_FRACTION": 1.0, - "GLOBAL_BATCH_SIZE": 128, - "LOOP_ADAPTER_ATTENTION_BACKEND": "sdpa_auto", - "LOOP_ADAPTER_CHECKPOINT_REFINER": true, - "LOOP_ADAPTER_FFN_SIZE": 5632, - "LOOP_FINAL_ADAPTER": "lookahead_refiner", - "MICRO_BATCH_SIZE": 4, - "ONLINE_DATASET": "HuggingFaceFW/fineweb-edu", - "ONLINE_DATASET_REVISION": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "ONLINE_DATASET_SUBSET": "sample-100BT", - "ONLINE_SOURCE_SPLIT": "train", - "ONLINE_SPLIT_SEED": "260421106", - "ONLINE_TOKENIZER_REVISION": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "ONLINE_VALIDATION_FRACTION": "0.001", - "PP": "1", - "PRECISION": "bf16", - "RUN_NAME": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "SAVE_INTERVAL": 1000, - "SEED": 42, - "SEQ_LENGTH": "2048", - "TP": "1", - "TRAIN_ITERS": 26703, - "WANDB_RUN_GROUP": "loop2_adapter_norm_sharekv_476m_7b_FBW_K3", - "WANDB_RUN_ID": "n8n2vo7q", - "WANDB_TAGS": "product,resume,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv" - }, - "eval_passes": 3, - "execution_architecture": "full_bandwidth_loop", - "feedback_enabled": true, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "ffn_hidden_size": 5632, - "final_validation": null, - "head_dim": 128, - "hidden_size": 1408, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "is_final_checkpoint": false, - "loading_code_revision": "3763dea5b6c0cb4496405eff8d7a83f103193769", - "loading_note": "Native Megatron tensor names and dtypes, weights only; no optimizer, RNG, scheduler or dataloader state. Not a Transformers AutoModel export.", - "logical_layers_per_pass": 40, - "logit_softcap": 15.0, - "loop_adapter": { - "attention_backend": "sdpa_auto", - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "checkpoint_refiner": true, - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "weight_sharing": "one adapter shared across feedback passes" - }, - "loop_final_adapter": "lookahead_refiner", - "metadata_source": "/root/dl/runtime/script/loopTF-workspace/output/launchings/20260918_loop476m_adapter_norm_sharekv/20260918-043531/product/training", - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "native_inference_adapter": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "max_supported_length": 2048, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": true, - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "num_passes": 3 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 8000 - }, - "native_loading_args": { - "architecture": "full_bandwidth_loop", - "attention_dropout": 0.0, - "aux_mtp_horizon": 1, - "backbone_profile": "paper2604_vanilla", - "context_parallel_size": 1, - "depth_num_loops": 2, - "depth_share_kv": true, - "feedback_eval_passes": 3, - "ffn_hidden_size": 5632, - "hidden_size": 1408, - "layernorm_epsilon": 1e-06, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": true, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "num_attention_heads": 11, - "num_layers": 20, - "num_query_groups": 11, - "padded_vocab_size": 32064, - "pipeline_model_parallel_size": 1, - "rotary_base": 10000, - "seq_length": 2048, - "tensor_model_parallel_size": 1, - "untie_embeddings_and_output_weights": true, - "vocab_size": 32001 - }, - "non_tensor_state_dict_entries": { - "decoder.stack.layers.0.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.0.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.1.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.10.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.11.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.12.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.13.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.14.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.15.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.16.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.17.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.18.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.19.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.2.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.3.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.4.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.5.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.6.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.7.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.8.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc1._extra_state": null, - "decoder.stack.layers.9.mlp.linear_fc2._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_proj._extra_state": null, - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state": null, - "feedback.state_proj._extra_state": null, - "feedback.token_gate_proj._extra_state": null, - "output_layer._extra_state": null - }, - "norm_epsilon": 1e-06, - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "num_attention_heads": 11, - "num_kv_heads": 11, - "num_layers": 20, - "objective": "latent_feedback_causal_lm", - "padded_vocab_size": 32064, - "parameter_counts": { - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808 - }, - "pipeline_parallel_size": 1, - "planned_training_tokens": 7000031232, - "projection_dtype": "bfloat16 (production)", - "qk_norm": "functional_rmsnorm_after_rope", - "residual_and_norm_scale_dtype": "float32", - "residual_multiplier": 1.0, - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "sequence_length": 2048, - "shared_layer_weights": true, - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "tensor_count": 150, - "tensor_dtypes": { - "torch.bfloat16": 101, - "torch.float32": 49 - }, - "tensor_parallel_size": 1, - "tied_embeddings": false, - "tokenizer": { - "active_vocab_size": 32001, - "additional_token": "<|assistant_end|>", - "repo_id": "NousResearch/Llama-2-7b-hf", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "total_parameters": 653383808, - "total_trainable_parameters": 653383808, - "train_passes": 3, - "training_code_diff_sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "training_code_dirty": true, - "training_code_revision": "f859c42e740ac6179c0ab383a0e5334cbf75b3f2", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531", - "training_tokens": 2097152000, - "vocab_size": 32001, - "width_scale": 22 -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/launch_config.env b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/launch_config.env deleted file mode 100644 index 0b4f43f9bde36b993cda6e1816e9071d91f9a775..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/launch_config.env +++ /dev/null @@ -1,114 +0,0 @@ -# Full-Bandwidth K3/L2 with three Look-ahead refiner blocks after each loop pair. -# Restore norm_recur -> norm_final after loop 2 and before the three refiners. -# Loop 2 computes fresh queries and reuses loop 1 KV at matching token positions. -# First-loop banks also feed the unchanged terminal adapter; gradients stay connected. -# See configs/loop/ADAPTER_CONTROL.md for normalization and comparison details. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=true -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_PACK_SEQUENCES=false # Adapter currently supports ordinary dense batches. -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Four-node successor to Look-ahead: preserve micro/global batch and the LR horizon. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=4 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# One accumulation round on 32 GPUs: 4 sequences x 32 GPUs = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# K=3 outer passes, each containing two shared-stack depth loops. -# CE objective = L1 + (L2 + L3) / 2; no intermediate depth-loop loss. -# 20 physical layers x 2 loops x 3 passes = 120 backbone + 9 refiner applications. -# Each final prediction uses refinement; one adapter is shared across passes. -FEEDBACK_PASS_MIXTURE=3:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=3 -FEEDBACK_LOSS_WEIGHT=1.0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop2_adapter_norm_sharekv_476m_7b_FBW_K3 -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,fullbandwidth,pass_k,FBW0,K3,lookahead_refiner,normalized_refiner_input,sharekv -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -FINAL_EVAL_ITERS=270 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/manifest.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/manifest.json deleted file mode 100644 index da519be2150d8e93a3412c715174113758f61471..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/manifest.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "checkpoint_iteration": 8000, - "files": { - "README.md": { - "sha256": "cd6a24a4557999a02c92ebb1e7276dc792c857978f3b63d537433c4deb642268", - "size": 2802 - }, - "config.json": { - "sha256": "d000d29bc0e4b6c52a423b1c95dcef20475ee5f5a0f24d2a169b1ce8f9212406", - "size": 13662 - }, - "launch_config.env": { - "sha256": "aae7032e8ed65417a5d673afa2e46a1941574342b39359b6473dbdb13b573ed6", - "size": 4135 - }, - "model.safetensors": { - "sha256": "6d2dd6cfd548f8b9f1b966056c460736f2265a10a336e225f0e09537559cfafd", - "size": 1306923800 - }, - "training_code.patch": { - "sha256": "babff6efa2f57b588f18f1e53426404f99efaefffe31e3e1a2b4e054a049c026", - "size": 23172 - }, - "training_recipe.json": { - "sha256": "fa2d683538194adde0b0f3bd192c459f6528f4a3bd28a4bad846f1e302c37557", - "size": 4716 - } - }, - "manifest_note": "Hashes cover other payload files; manifest does not hash itself.", - "path_in_repo": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000", - "repo_id": "Siyuc/loopTF-workspace", - "repo_type": "model", - "training_run": "paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531" -} diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/model.safetensors b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/model.safetensors deleted file mode 100644 index 86dc81d516a3715f92ed3354ee083693304f4e8a..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6d2dd6cfd548f8b9f1b966056c460736f2265a10a336e225f0e09537559cfafd -size 1306923800 diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_code.patch b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_code.patch deleted file mode 100644 index 11e95c586255c0699577301bd07761599a010153..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_code.patch +++ /dev/null @@ -1,357 +0,0 @@ -diff --git a/configs/loop/README.md b/configs/loop/README.md -index 9781c2a..4df1583 100644 ---- a/configs/loop/README.md -+++ b/configs/loop/README.md -@@ -1,5 +1,10 @@ - # Shared-stack loop controls for Look-ahead K3/LA2 - -+The [normalized adapter with KV sharing](KV_SHARING.md) adds an optional -+second-loop read of first-loop KV while retaining the three terminal refiners. -+Its four-node [configuration](476M_7b_FBW0_K3_L2_ADAPTER_SHARE_KV.env) matches the -+finishing Look-ahead allocation and preserves the 7B-token budget. -+ - The additional **[three-block refiner control](ADAPTER_CONTROL.md)** is - `476M_7b_FBW0_K3_L2_ADAPTER.env`: the second loop's final hidden state feeds - three Look-ahead refiner blocks after the original RMSNorm pair, with no fusion. It preserves K3 and -diff --git a/scripts/evaluate_benchmarks.sh b/scripts/evaluate_benchmarks.sh -index dfb7fd1..b0407dc 100644 ---- a/scripts/evaluate_benchmarks.sh -+++ b/scripts/evaluate_benchmarks.sh -@@ -1,5 +1,6 @@ - #!/usr/bin/env bash --# Does not allocate GPUs, stop jobs, execute generated code or enable W&B. -+# Plan by default. Paper2604 execution accepts an explicit device and launch card. -+# Resource handoff for a CUDA launch is the caller's responsibility. - set -euo pipefail - SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" - source "${SCRIPT_DIR}/workspace_env.sh" -diff --git a/scripts/launch_megatron_multinode.sh b/scripts/launch_megatron_multinode.sh -index e4f3d64..868360c 100755 ---- a/scripts/launch_megatron_multinode.sh -+++ b/scripts/launch_megatron_multinode.sh -@@ -176,6 +176,7 @@ for index in "${!HOSTS[@]}"; do - # mismatch between discovery order and the allocation cannot misrank a node. - remote_command=$(printf 'set -Eeuo pipefail - cd %q -+export PROJECT_ROOT=%q - if [[ -r scripts/pod_profile.local.sh ]]; then - source scripts/pod_profile.local.sh - fi -@@ -199,7 +200,7 @@ export HF_HOME="${HF_HOME:-%q/data/cache/huggingface}" - if [[ -r /root/dl/runtime/script/.env ]]; then set -a; . /root/dl/runtime/script/.env; set +a; fi - printf "node_rank=%%s nnodes=%%s host=%%s\\n" "${NODE_RANK}" "${NNODES}" "$(hostname)" - exec bash scripts/train_full_bandwidth_megatron.sh --config %q %s' \ -- "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ -+ "$PROJECT_ROOT" "$PROJECT_ROOT" "$NNODES" "$MASTER_ADDR" "$MASTER_PORT" "$index" \ - "$PROXY" "$PROXY" "$MEGATRON_LM_PATH" "$TRANSFORMERS_PATH" \ - "$TORCHRUN_BIN" "$GPU_RESOURCE_MANAGER" \ - "${HF_HUB_ETAG_TIMEOUT:-60}" "${HF_HUB_DOWNLOAD_TIMEOUT:-120}" \ -diff --git a/scripts/train_full_bandwidth_megatron.sh b/scripts/train_full_bandwidth_megatron.sh -index 90684c1..116c071 100755 ---- a/scripts/train_full_bandwidth_megatron.sh -+++ b/scripts/train_full_bandwidth_megatron.sh -@@ -939,7 +939,7 @@ export ONLINE_PREFETCH_BATCHES ONLINE_SOURCE_MODE ONLINE_RAW_MANIFEST ONLINE_PAC - export ONLINE_LATENCY_LOG_DIR ONLINE_VALIDATION_MANIFEST EXIT_INTERVAL - export WANDB_ENTITY WANDB_PROJECT WANDB_EXP_NAME WANDB_SAVE_DIR WANDB_RESUME - export WANDB_MODE DISABLE_WANDB TENSORBOARD_DIR --env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ -+env | grep -E '^(LOOP_[A-Z0-9_]+|LOOKAHEAD_[A-Z0-9_]+|AUX_MTP_[A-Z0-9_]+|HUGINN_[A-Z0-9_]+|BACKBONE_PROFILE|LAYERNORM_EPSILON|UNTIE_EMBEDDINGS|RESUME_POLICY|REQUIRE_CHECKPOINT|GROUP_QUERY_ATTENTION|QK_LAYERNORM|ATTENTION_OUTPUT_GATE|SWIGLU|DISABLE_BIAS_LINEAR|TRAINING_RECIPE|RECIPE_[A-Z0-9_]+|ADAM_[A-Z0-9_]+|CLIP_GRAD|PAPER_[A-Z0-9_]+|DATA_ROOT|OUTPUT_ROOT|DATASET_NAME|DATA_DIR|DATA_MANIFEST|RUN_NAME|RUN_DIR|TRAIN_ARCHITECTURE|DEPTH_NUM_LOOPS|DEPTH_SHARE_KV|WANDB_(ENTITY|PROJECT|EXP_NAME|SAVE_DIR|RUN_ID|RESUME|MODE|TAGS|RUN_GROUP)|DISABLE_WANDB|TENSORBOARD_DIR|SEED|MICRO_BATCH_SIZE|GLOBAL_BATCH_SIZE|TRAIN_ITERS|WARMUP_ITERS|WSD_DECAY_ITERS|SAVE_INTERVAL|SAVE_RETAIN_INTERVAL|EXIT_INTERVAL|EVAL_INTERVAL|EVAL_ITERS|FINAL_EVAL_ITERS|EVAL_AT_START|EVAL_ONLY|LOG_INTERVAL|CKPT_FORMAT|TOKENIZER_MODEL|TRAIN_DATA_PATH|VALID_DATA_PATH|TEST_DATA_PATH|SAVE_PATH|LOAD_PATH|DATALOADER_SAVE|GPUS_PER_NODE|TP|PP|NUM_LAYERS|HIDDEN_SIZE|FFN_HIDDEN_SIZE|NUM_ATTENTION_HEADS|NUM_QUERY_GROUPS|SEQ_LENGTH|MAX_POSITION_EMBEDDINGS|NORMALIZATION|HIDDEN_DROPOUT|ATTENTION_DROPOUT|WINDOW_SIZE|WINDOW_ATTN_SKIP_FREQ|POSITION_EMBEDDING_TYPE|ROTARY_BASE|TOKENIZER_TYPE|VOCAB_SIZE|MAKE_VOCAB_SIZE_DIVISIBLE_BY|NUM_WORKERS|PRECISION|ENABLE_RECOMPUTE|RECOMPUTE_[A-Z0-9_]+|LR|MIN_LR|LR_DECAY_STYLE|LR_WSD_DECAY_STYLE|OPTIMIZER|MUON_MOMENTUM|MUON_NESTEROV|WEIGHT_DECAY|FEEDBACK_[A-Z0-9_]+|MEGATRON_EXTRA_ARGS|DATA_PREFLIGHT|EXPECTED_RECORD_LENGTH|EXPECTED_INPUT_LENGTH|MIN_MODEL_INPUT_TOKENS|ONLINE_[A-Z0-9_]+)=' \ - | sort > "${RUN_DIR}/launch_env.node${NODE_RANK}.txt" - - export MEGATRON_LM_PATH -diff --git a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -index aae62fa..e67a9f0 100644 ---- a/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -+++ b/src/full_bandwidth_loop_tf/megatron/adapter_decoder.py -@@ -14,12 +14,12 @@ from .decoder import LoopedDecoder - - - class AdapterLoopedDecoder(LoopedDecoder): -- def __init__(self, decoder, loop_config, adapter_config): -+ def __init__(self, decoder, loop_config, adapter_config, *, share_kv=False): - adapter_config.validate_model( - profile="paper2604_vanilla", loops=loop_config.num_loops, - layers=len(decoder.layers), - ) -- super().__init__(decoder, loop_config) -+ super().__init__(decoder, loop_config, share_kv=share_kv) - config = self.config - self.bank_layers = (0, len(self.layers) // 2, len(self.layers) - 1) - self.adapter = LoopFinalAdapter(SimpleNamespace( -@@ -70,17 +70,35 @@ class AdapterLoopedDecoder(LoopedDecoder): - value.permute(1, 2, 0, 3)) - return hook - -- try: -- for index in self.bank_layers: -- core = self.layers[index].self_attention.core_attention -- handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -- first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -- finally: -- for handle in handles: -- handle.remove() -- if set(banks) != set(self.bank_layers): -- raise RuntimeError("not all first-loop KV banks were captured") -- second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ if self.share_kv: -+ try: -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = False -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ for index in self.bank_layers: -+ bank = self.layers[index].self_attention.depth_kv_bank -+ # Borrow the same projected tensors used by loop 2. No -+ # detached copies, new projections, or second RoPE rotation. -+ banks[index] = tuple(x.reshape(batch, length, *x.shape[1:]).transpose(1, 2) -+ for x in bank) -+ for layer in self.layers: -+ layer.self_attention.depth_reuse_kv = True -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) -+ finally: -+ for layer in self.layers: -+ layer.self_attention.clear_depth_kv() -+ else: -+ try: -+ for index in self.bank_layers: -+ core = self.layers[index].self_attention.core_attention -+ handles.append(core.register_forward_pre_hook(capture(index), with_kwargs=True)) -+ first = self.stack(hidden_states=hidden_states, attention_mask=attention_mask, **kwargs) -+ finally: -+ for handle in handles: -+ handle.remove() -+ if set(banks) != set(self.bank_layers): -+ raise RuntimeError("not all first-loop KV banks were captured") -+ second = self.stack(hidden_states=first, attention_mask=attention_mask, **kwargs) - # Normalize the second-loop residual stream before the three refiners. - # Retain the original norm_recur -> norm_final pair, once per FBW pass. - result = self.adapter( -diff --git a/src/full_bandwidth_loop_tf/megatron/attention.py b/src/full_bandwidth_loop_tf/megatron/attention.py -index b24fc06..b7c7984 100644 ---- a/src/full_bandwidth_loop_tf/megatron/attention.py -+++ b/src/full_bandwidth_loop_tf/megatron/attention.py -@@ -9,6 +9,17 @@ class SharedKVSelfAttention(SelfAttention): - depth_kv_bank = None - depth_reuse_kv = False - -+ def __init__(self, *args, **kwargs): -+ super().__init__(*args, **kwargs) -+ if self.config.tensor_model_parallel_size != 1: -+ raise ValueError("depth KV sharing requires TP=1") -+ for name in ("gradient_accumulation_fusion", "recompute_granularity", "sequence_parallel", -+ "attention_output_gate", "add_bias_linear", "add_qkv_bias", "qk_layernorm"): -+ if getattr(self.config, name, None): -+ raise ValueError(f"depth KV sharing does not support {name}") -+ if self.num_query_groups_per_partition != self.num_attention_heads_per_partition: -+ raise ValueError("depth KV sharing requires one KV head per query head") -+ - def clear_depth_kv(self): - self.depth_kv_bank = None - self.depth_reuse_kv = False -diff --git a/src/full_bandwidth_loop_tf/megatron/config.py b/src/full_bandwidth_loop_tf/megatron/config.py -index 7e26569..f6dd960 100644 ---- a/src/full_bandwidth_loop_tf/megatron/config.py -+++ b/src/full_bandwidth_loop_tf/megatron/config.py -@@ -92,8 +92,6 @@ def build_model_config(args): - - - def validate_kv_sharing(args): -- if getattr(args, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if getattr(args, "backbone_profile", None) != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - if args.depth_num_loops < 2: -diff --git a/src/full_bandwidth_loop_tf/megatron/model.py b/src/full_bandwidth_loop_tf/megatron/model.py -index 4ccad16..cb5dadf 100644 ---- a/src/full_bandwidth_loop_tf/megatron/model.py -+++ b/src/full_bandwidth_loop_tf/megatron/model.py -@@ -13,8 +13,6 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - def __init__(self, *args, feedback_config, **kwargs) -> None: - share_kv = getattr(feedback_config, "depth_share_kv", False) - if share_kv: -- if getattr(feedback_config, "loop_final_adapter", "none") != "none": -- raise ValueError("depth KV sharing does not support a loop final adapter") - if feedback_config.backbone_profile != "paper2604_vanilla": - raise ValueError("depth KV sharing requires paper2604_vanilla") - from .attention import SharedKVSelfAttention -@@ -33,4 +31,4 @@ class LoopedFullBandwidthMegatronModel(FullBandwidthMegatronModel): - else: - from .adapter_decoder import AdapterLoopedDecoder - -- self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter) -+ self.decoder = AdapterLoopedDecoder(self.decoder, loops, adapter, share_kv=share_kv) -diff --git a/src/full_bandwidth_tf/launch.py b/src/full_bandwidth_tf/launch.py -index 5f1b5f5..c838db8 100644 ---- a/src/full_bandwidth_tf/launch.py -+++ b/src/full_bandwidth_tf/launch.py -@@ -21,6 +21,14 @@ def launcher_backbone_metadata(env): - FEEDBACK_PROGRESSIVE_STAGES="[]") - - profile = env.get("BACKBONE_PROFILE", FBT_PROFILE) -+ share = env.get("DEPTH_SHARE_KV", "false") -+ if share not in {"true", "false"}: -+ raise ValueError("DEPTH_SHARE_KV must be true or false") -+ if share == "true": -+ if profile != PAPER_PROFILE or env.get("TRAIN_ARCHITECTURE") != "full_bandwidth_loop": -+ raise ValueError("depth KV sharing requires full_bandwidth_loop and paper2604_vanilla") -+ if env.get("ONLINE_PACK_SEQUENCES", "false") != "false": -+ raise ValueError("depth KV sharing requires dense unpacked batches") - if profile == FBT_PROFILE: - return { - "execution_architecture": env.get("TRAIN_ARCHITECTURE", "full_bandwidth"), -@@ -85,9 +93,6 @@ def launcher_backbone_metadata(env): - args = SimpleNamespace(**values) - if args.architecture == "full_bandwidth_loop": - args.depth_num_loops = int(env.get("DEPTH_NUM_LOOPS", "2")) -- share = env.get("DEPTH_SHARE_KV", "false") -- if share not in {"true", "false"}: -- raise ValueError("DEPTH_SHARE_KV must be true or false") - args.depth_share_kv = share == "true" - if args.depth_share_kv and args.depth_num_loops < 2: - raise ValueError("depth KV sharing requires at least two loops") -diff --git a/src/loop_transformer_backends/benchmarks/cli.py b/src/loop_transformer_backends/benchmarks/cli.py -index 718eebe..61977fa 100644 ---- a/src/loop_transformer_backends/benchmarks/cli.py -+++ b/src/loop_transformer_backends/benchmarks/cli.py -@@ -1,7 +1,7 @@ --"""Plan ten IFM task families; optionally smoke-test an exported HF model on CPU. -+"""Plan or run downstream benchmarks: IFM coverage and paper2604 continuation loss. - --No Megatron checkpoint conversion, GPU management, downloads or generated-code --execution is implicit. Planning imports neither torch nor evaluation packages. -+No model loading, GPU management, downloads or generated-code execution is -+implicit. Planning imports neither torch nor evaluation packages. - """ - - from __future__ import annotations -@@ -31,10 +31,22 @@ def within(path: Path, roots: list[Path]) -> Path: - def read_suite(path: Path) -> dict: - suite = json.loads(path.read_text()) - names = [task["name"] for task in suite["tasks"]] -+ if not names or any(not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", n) for n in names): -+ raise ValueError("Suite task names must be nonempty safe identifiers") - if suite["schema_version"] != 1 or len(set(names)) != len(names): - raise ValueError("Unsupported schema or duplicate task names") -- if any(t["engine"] not in {"lm_eval", "evalplus"} for t in suite["tasks"]): -+ if any(t["engine"] not in {"lm_eval", "evalplus", "paper2604"} for t in suite["tasks"]): - raise ValueError("Unknown evaluation engine") -+ if any(t["engine"] == "paper2604" for t in suite["tasks"]): -+ if not all(t["engine"] == "paper2604" for t in suite["tasks"]): -+ raise ValueError("Paper2604 must use its own suite") -+ for task in suite["tasks"]: -+ if task["task_type"] not in {"language_modeling", "multiple_choice"}: -+ raise ValueError("Unsupported paper2604 task type") -+ if type(task["fewshot"]) is not int or task["fewshot"] < 0: -+ raise ValueError("Paper2604 requires an explicit nonnegative shot count") -+ if task["bundle"] not in {"core", "saunshi", "owned"}: -+ raise ValueError("Unknown paper2604 data bundle") - return suite - - -@@ -64,6 +76,8 @@ def commands(suite: dict, model: Path, run: Path, *, limit: int | None) -> list[ - defaults = suite["defaults"] - result = [] - for task in suite["tasks"]: -+ if task["engine"] == "paper2604": -+ raise ValueError("Use the dedicated paper2604 runner for this suite") - dest = run / task["name"] - if task["engine"] == "lm_eval": - argv = [ -@@ -111,10 +125,10 @@ def validate_hf_model(model: Path) -> None: - raise ValueError("Custom remote-code model loading is not enabled") - - --def validate_card(card: Path, project: Path) -> None: -- relative = card.resolve().relative_to((project / "launchings" / "debug").resolve()) -- if len(relative.parts) != 2 or card.suffix != ".md": -- raise ValueError("A debug launching card in an experiment group is required") -+def validate_card(card: Path, project: Path, *, categories=("debug",)) -> None: -+ relative = card.resolve().relative_to((project / "launchings").resolve()) -+ if len(relative.parts) != 3 or relative.parts[0] not in categories or card.suffix != ".md": -+ raise ValueError(f"A {categories} launching card in an experiment group is required") - text = card.read_text() - for field in ["Purpose:", "Running branch:", "Git commit:", "Config:", - "Architecture:", "Dataset:", "Node IPs:", "GPU specification:"]: -@@ -129,20 +143,45 @@ def main() -> None: - parser.add_argument("--suite", type=Path, default=PROJECT / "configs/benchmarks/ifm_v1.json") - parser.add_argument("--list", action="store_true") - parser.add_argument("--tasks", default="all", help="Comma-separated names from --list") -- parser.add_argument("--model", type=Path, help="Local standard HF export under data/ or output/") -+ parser.add_argument("--model", type=Path, -+ help="Local model under data/ or output/; paper2604 also accepts native .pt") - parser.add_argument("--run-id", help="Unique artifact namespace; never overwrite another attempt") - parser.add_argument("--launch-card", type=Path) - parser.add_argument("--execute", action="store_true", -- help="Execute CPU lm_eval tasks only; never code or GPU workloads") -+ help="Execute lm_eval on CPU, or paper2604 on the explicitly selected device") - parser.add_argument("--allow-network", action="store_true", - help="Explicitly allow dataset network reads; default is offline") - parser.add_argument("--limit", type=int, help="Positive examples/task limit; debug only") -+ parser.add_argument("--prepare", action="store_true", help="Build/import a paper2604 bundle") -+ parser.add_argument("--bundle", type=Path, help="Frozen paper2604 bundle under DATA_ROOT") -+ parser.add_argument("--source-bundles", type=Path, -+ help="Import authors' eval_bundle/saunshi_bundle/owned_bundle directories") -+ parser.add_argument("--allow-count-mismatch", action="store_true", -+ help="Allow different dataset counts during debug preparation only") -+ parser.add_argument("--tokenizer", type=Path, help="Local raw tokenizer.json directory") -+ parser.add_argument("--adapter", choices=("native", "hf"), default="native") -+ parser.add_argument("--device", default="cpu", help="Paper2604 device; default CPU") -+ parser.add_argument("--dtype", choices=("checkpoint", "float32", "bfloat16"), -+ default="checkpoint") -+ parser.add_argument("--max-length", type=int, help="Paper2604 explicit context limit") -+ parser.add_argument("--context-policy", choices=("error", "left-truncate"), default="error") -+ parser.add_argument("--megatron-lm-path", default=os.environ.get("MEGATRON_LM_PATH")) - args = parser.parse_args() -+ if str(args.suite) == "paper2604": -+ args.suite = PROJECT / "configs/benchmarks/paper2604_v1.json" - suite = read_suite(args.suite) - if args.list: - for task in suite["tasks"]: - print(f"{task['name']:18} {task['engine']:9} {task['task']}") - return -+ if suite["tasks"] and suite["tasks"][0]["engine"] == "paper2604": -+ from .paper2604.runner import run -+ run(args, suite, parser) -+ return -+ if args.prepare or args.bundle or args.source_bundles or args.tokenizer: -+ parser.error("Paper2604 options require --suite paper2604") -+ if args.device != "cpu": -+ parser.error("The existing lm_eval runner supports CPU execution only") - if not args.model or not args.run_id: - parser.error("--model and --run-id are required unless --list") - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", args.run_id): -diff --git a/src/loop_transformer_backends/megatron/runtime.py b/src/loop_transformer_backends/megatron/runtime.py -index fc03aaf..74ef805 100644 ---- a/src/loop_transformer_backends/megatron/runtime.py -+++ b/src/loop_transformer_backends/megatron/runtime.py -@@ -176,6 +176,10 @@ def run( - model_config = architecture.build_model_config(args) - with ExitStack() as hooks: - hooks.callback(install_ft_args_hook(args)) -+ if getattr(args, "wandb_project", None): -+ from .wandb_hashing import install_wandb_hashing_hook -+ -+ hooks.callback(install_wandb_hashing_hook()) - if args.training_recipe == DEFAULT_RECIPE: - hooks.enter_context(paper_checkpoint_context()) - hooks.callback(install_paper_hooks(args)) -diff --git a/tests/test_paper2604_backbone.py b/tests/test_paper2604_backbone.py -index ebb118d..059f44c 100644 ---- a/tests/test_paper2604_backbone.py -+++ b/tests/test_paper2604_backbone.py -@@ -264,6 +264,8 @@ def test_production_preflight_geometry_recipe_logging_and_fresh_run(tmp_path): - assert evaluation_metrics({"pass 1 loss": 2.3}, "val", 1).keys() == { - "val/pass_1_ce", - "val/pass_1_ppl", -+ "val/final_ce", -+ "val/final_ppl", - } - assert training_metric_name("pass 1 loss") == "train/pass_1_ce" - (run / "checkpoints").mkdir() -@@ -331,6 +333,7 @@ def test_actual_production_command_has_no_feedback_flags_or_checkpoint_load(tmp_ - capture = tmp_path / "args.txt" - env = { - "CAPTURE_ARGS": str(capture), -+ "TORCHRUN_BIN": str(spy), - "PATH": f"{tmp_path}:{Path(sys.executable).parent}:{os.environ['PATH']}", - } - result = launch(tmp_path, f"GPU_RESOURCE_MANAGER={manager}", preflight=False, extra_env=env) diff --git a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_recipe.json b/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_recipe.json deleted file mode 100644 index e8b67e5fa25e9e1b0865c8d27df9a3455d1165c0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_FBW0_K3_adapter_norm_sharekv_20260918-043531/step_8000/training_recipe.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 653383808, - "feedback_parameters": 3967744, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "latent_feedback_causal_lm", - "feedback_enabled": true, - "train_passes": 3, - "eval_passes": 3, - "feedback_pass_mixture": { - "3": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": true, - "depth_kv_policy": "first_loop_per_layer_connected", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": true, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 6.449999999999999 - } -} diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md deleted file mode 100644 index 686b59f75f199fbfb89bcda562081b72b115a3b0..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# L2 Adapter IFM final weights - -Final step 26,703 native weights-only export for the paper2604 L2 loop with IFM diagonal input write and terminal Look-ahead adapter. - -`model.safetensors` is a native Megatron state dict, not a Transformers AutoModel checkpoint. It excludes optimizer, scheduler, RNG, dataloader, logs, datasets, and credentials. Use the recorded full_bandwidth_loop configuration and strict loading; `config.json` records native reconstruction metadata. - -Use `manifest.json` to verify payload hashes. diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json deleted file mode 100644 index 7971349515912a540b33d0c1335aeaeca60d5a20..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json +++ /dev/null @@ -1,1117 +0,0 @@ -{ - "architecture": "full_bandwidth_loop", - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_version": 3.0, - "excluded_null_extra_state_keys": [ - "decoder.stack.layers.0.self_attention.linear_proj._extra_state", - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.0.mlp.linear_fc1._extra_state", - "decoder.stack.layers.0.mlp.linear_fc2._extra_state", - "decoder.stack.layers.1.self_attention.linear_proj._extra_state", - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.1.mlp.linear_fc1._extra_state", - "decoder.stack.layers.1.mlp.linear_fc2._extra_state", - "decoder.stack.layers.2.self_attention.linear_proj._extra_state", - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.2.mlp.linear_fc1._extra_state", - "decoder.stack.layers.2.mlp.linear_fc2._extra_state", - "decoder.stack.layers.3.self_attention.linear_proj._extra_state", - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.3.mlp.linear_fc1._extra_state", - "decoder.stack.layers.3.mlp.linear_fc2._extra_state", - "decoder.stack.layers.4.self_attention.linear_proj._extra_state", - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.4.mlp.linear_fc1._extra_state", - "decoder.stack.layers.4.mlp.linear_fc2._extra_state", - "decoder.stack.layers.5.self_attention.linear_proj._extra_state", - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.5.mlp.linear_fc1._extra_state", - "decoder.stack.layers.5.mlp.linear_fc2._extra_state", - "decoder.stack.layers.6.self_attention.linear_proj._extra_state", - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.6.mlp.linear_fc1._extra_state", - "decoder.stack.layers.6.mlp.linear_fc2._extra_state", - "decoder.stack.layers.7.self_attention.linear_proj._extra_state", - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.7.mlp.linear_fc1._extra_state", - "decoder.stack.layers.7.mlp.linear_fc2._extra_state", - "decoder.stack.layers.8.self_attention.linear_proj._extra_state", - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.8.mlp.linear_fc1._extra_state", - "decoder.stack.layers.8.mlp.linear_fc2._extra_state", - "decoder.stack.layers.9.self_attention.linear_proj._extra_state", - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.9.mlp.linear_fc1._extra_state", - "decoder.stack.layers.9.mlp.linear_fc2._extra_state", - "decoder.stack.layers.10.self_attention.linear_proj._extra_state", - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.10.mlp.linear_fc1._extra_state", - "decoder.stack.layers.10.mlp.linear_fc2._extra_state", - "decoder.stack.layers.11.self_attention.linear_proj._extra_state", - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.11.mlp.linear_fc1._extra_state", - "decoder.stack.layers.11.mlp.linear_fc2._extra_state", - "decoder.stack.layers.12.self_attention.linear_proj._extra_state", - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.12.mlp.linear_fc1._extra_state", - "decoder.stack.layers.12.mlp.linear_fc2._extra_state", - "decoder.stack.layers.13.self_attention.linear_proj._extra_state", - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.13.mlp.linear_fc1._extra_state", - "decoder.stack.layers.13.mlp.linear_fc2._extra_state", - "decoder.stack.layers.14.self_attention.linear_proj._extra_state", - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.14.mlp.linear_fc1._extra_state", - "decoder.stack.layers.14.mlp.linear_fc2._extra_state", - "decoder.stack.layers.15.self_attention.linear_proj._extra_state", - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.15.mlp.linear_fc1._extra_state", - "decoder.stack.layers.15.mlp.linear_fc2._extra_state", - "decoder.stack.layers.16.self_attention.linear_proj._extra_state", - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.16.mlp.linear_fc1._extra_state", - "decoder.stack.layers.16.mlp.linear_fc2._extra_state", - "decoder.stack.layers.17.self_attention.linear_proj._extra_state", - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.17.mlp.linear_fc1._extra_state", - "decoder.stack.layers.17.mlp.linear_fc2._extra_state", - "decoder.stack.layers.18.self_attention.linear_proj._extra_state", - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.18.mlp.linear_fc1._extra_state", - "decoder.stack.layers.18.mlp.linear_fc2._extra_state", - "decoder.stack.layers.19.self_attention.linear_proj._extra_state", - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.19.mlp.linear_fc1._extra_state", - "decoder.stack.layers.19.mlp.linear_fc2._extra_state", - "output_layer._extra_state" - ], - "excluded_state": [ - "optimizer", - "opt_param_scheduler", - "rng_state", - "dataloader" - ], - "iteration": 26703, - "loading": { - "format": "native Megatron model state; not a Transformers AutoModel", - "instructions": "Use loop_transformer_backends.benchmarks.models.build_native on the original checkpoint schema, or reconstruct the recorded full_bandwidth_loop/IFM/adapter architecture before strict state loading." - }, - "model_arguments": { - "_is_global_batch_size_explicitly_specified": true, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "accumulate_allreduce_grads_in_fp32": true, - "activation_func_clamp_value": null, - "activation_offload_fraction": 1.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "adaptive_muon_moment2_method": "normuon", - "add_bias_linear": false, - "add_position_embedding": true, - "add_qkv_bias": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "align_grad_reduce": true, - "align_param_gather": false, - "allow_ambiguous_pad_tokens": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "apply_query_key_layer_scaling": false, - "apply_residual_connection_post_layernorm": false, - "apply_rope_fusion": false, - "apply_wd_to_qk_layernorm": false, - "architecture": "full_bandwidth_loop", - "architecture_module": "[]", - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "async_save": false, - "async_strategy": "mcore", - "attention_backend": "AttnBackend.auto", - "attention_cp_layout": "zigzag", - "attention_dropout": 0.0, - "attention_output_gate": false, - "attention_softmax_in_fp32": false, - "auto_detect_ckpt_format": false, - "aux_mtp_horizon": 1, - "aux_mtp_loss_weight": 0.3, - "aux_mtp_rank": 128, - "backbone_metadata": "{'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 651401344, 'feedback_parameters': 0, 'loop_input_write_parameters': 1985280, 'loop_adapter_parameters': 83271936}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'full_bandwidth_loop', 'objective': 'depth_loop_causal_lm', 'feedback_enabled': False, 'train_passes': 1, 'eval_passes': 1, 'feedback_pass_mixture': {1: 1.0}, 'feedback_progressive_stages': [], 'depth_num_loops': 2, 'logical_layers_per_pass': 40, 'shared_layer_weights': True, 'depth_share_kv': False, 'depth_kv_policy': 'recompute_each_loop', 'norm_placement': 'embedding once; recur/final after loop 2 before three refiners; no post-refiner norm', 'loop_input_write': 'ifm_diagonal', 'loop_input_write_source': 'raw_token_embeddings', 'loop_input_write_placement': 'before_every_depth_loop_after_initial_norm_emb', 'loop_input_write_sharing': 'all_depth_loops_and_outer_feedback_passes', 'loop_input_write_init': 'paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)', 'loop_final_adapter': 'lookahead_refiner', 'loop_adapter': {'ffn_size': 5632, 'state_input': 'second-loop hidden states after norm_recur and norm_final', 'after_depth_loop': 2, 'applications_per_pass': 1, 'terminal_norms': True, 'attention_backend': 'sdpa_auto', 'checkpoint_refiner': False, 'bank_layers': [0, 10, 19], 'bank_source': 'first depth loop, fresh on each feedback pass', 'weight_sharing': 'one adapter shared across feedback passes'}, 'block_application_multiplier_per_pass': 2.15, 'block_application_multiplier': 2.15}", - "backbone_profile": "paper2604_vanilla", - "barrier_with_L1_time": true, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "batch_invariant_mode": false, - "bert_binary_head": true, - "bert_embedder_type": "megatron", - "bert_load": null, - "bf16": true, - "bias_dropout_fusion": false, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "block_data_path": null, - "cache_mla_latents": false, - "calc_ft_timeouts": false, - "calculate_per_token_loss": false, - "chat_template": null, - "check_for_large_grads": false, - "check_for_nan_in_loss_and_grad": true, - "check_for_spiky_loss": false, - "check_weight_hash_across_dp_replicas_interval": null, - "ckpt_assume_constant_structure": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_drop_redundant_extra_state": false, - "ckpt_format": "torch", - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_fully_parallel_save": true, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_load_validate_sharding_integrity": true, - "ckpt_pg_tensors_cache_create": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_step": null, - "classes_fraction": 1.0, - "clip_grad": 0.0, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "consumed_train_samples": 3417984, - "consumed_valid_samples": 68608, - "context_parallel_size": 1, - "cp_comm_type": "['p2p']", - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "create_all_gather_group": false, - "create_attention_mask_in_dataloader": false, - "cross_entropy_fusion_impl": "native", - "cross_entropy_loss_fusion": false, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "csa_window_size": 128, - "cuda_graph_impl": "none", - "cuda_graph_modules": "[]", - "cuda_graph_warmup_steps": 3, - "curr_iteration": 26702, - "data_args_path": null, - "data_cache_path": null, - "data_parallel_random_init": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "data_parallel_size": 16, - "data_path": null, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "dataloader_defer_npy_index_mmap": false, - "dataloader_fast_cache_load": false, - "dataloader_inter_document_masking": false, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/dataloader", - "dataloader_type": "external", - "ddp_average_in_collective": false, - "ddp_bucket_size": null, - "ddp_num_buckets": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": "[]", - "ddp_reduce_scatter_with_fp32_accumulation": false, - "decode_only_cuda_graphs": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "decoder_num_layers": null, - "decoder_seq_length": null, - "decoupled_lr": null, - "decoupled_min_lr": null, - "decrease_batch_size_if_needed": false, - "defer_embedding_wgrad_compute": false, - "delay_offload_until_cuda_graph": false, - "delay_wgrad_compute": false, - "delta_offload_bytes_across_pp_ranks": 0, - "deprecated_use_mcore_models": false, - "depth_num_loops": 2, - "depth_share_kv": false, - "deterministic_mode": false, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_head_hidden_size": 2048, - "dino_local_crops_number": 10, - "dino_local_img_size": 96, - "dino_norm_last_layer": false, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp": 0.04, - "dino_warmup_teacher_temp_epochs": 30, - "disable_bf16_reduced_precision_matmul": false, - "disable_jit_fuser": false, - "disable_msc_deprecated": false, - "disable_straggler_on_startup": false, - "disable_symmetric_registration": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_optim_fully_reshardable": false, - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_workers": 1, - "distrib_optim_fully_reshardable_mem_efficient": false, - "distribute_saved_activations": false, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "distributed_timeout_seconds_after_init": null, - "do_test": 0, - "do_train": 1, - "do_valid": 1, - "dsa_indexer_head_dim": null, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dump_param_to_param_group_map": null, - "embedding_init_method_std": null, - "embedding_path": null, - "empty_unused_memory_level": 0, - "enable_chunked_prefill": false, - "enable_cuda_graph": false, - "enable_experimental": false, - "enable_ft_package": false, - "enable_full_sharding_in_hsdp": false, - "enable_mhc_connections": false, - "enable_msc": false, - "enable_one_logger": true, - "encoder_num_layers": 20, - "encoder_seq_length": 2048, - "end_weight_decay": 0.0, - "eod_mask_loss": false, - "ep_overlap_early_attn_memory_release": false, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "eval_at_start": true, - "eval_global_batch_size": 128, - "eval_interval": 100, - "eval_iters": 2, - "eval_micro_batch_size": 4, - "evaluation_metadata": "{'eval_at_start': True, 'initial_iters': 2, 'intermediate_iters': 2, 'final_validation_iters': 270, 'test_iters': 0, 'global_batch_size': 128, 'sequence_length': 2048, 'intermediate_input_tokens': 524288, 'final_validation_sequences': 34560, 'final_validation_input_tokens': 70778880, 'validation_iterator': 'continues_existing_held_out_iterator'}", - "evidence_data_path": null, - "exit_duration_in_mins": null, - "exit_interval": null, - "exit_on_missing_checkpoint": false, - "exit_signal": 15, - "exit_signal_handler": false, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "exp_avg_dtype": "torch.float32", - "exp_avg_sq_dtype": "torch.float32", - "experimental_attention_variant": null, - "expert_data_parallel_sharding_strategy": null, - "expert_gtp_weight_remat_size": 1, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "expert_tensor_parallel_size": 1, - "external_cuda_graph": false, - "fake_process_group": false, - "fault_injector_delay_start_iteration": null, - "fault_injector_fault_delay": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_types": null, - "fault_injector_mtti_seconds": null, - "fault_injector_num_ranks": null, - "fault_injector_offset_seconds": null, - "fault_injector_ranks": null, - "fault_injector_seed": null, - "feedback_eval_passes": 1, - "feedback_jitter_sigma": 0.0, - "feedback_layers": null, - "feedback_log_hidden_change": false, - "feedback_loss_weight": 0.0, - "feedback_normuon_beta2": 0.95, - "feedback_pass_mixture": "1:1.0", - "feedback_prefix_mixin": false, - "feedback_progressive_stages": "[]", - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "ffn_hidden_size": 5632, - "fim_data": false, - "fim_eod_token": "<|endoftext|>", - "fim_fragment_rate": null, - "fim_middle_token": "", - "fim_no_prefix": null, - "fim_pad_token": "", - "fim_prefix_token": "", - "fim_rate": 0.5, - "fim_split_sample": null, - "fim_spm_rate": 0.5, - "fim_suffix_token": "", - "final_eval_iters": 270, - "fine_grained_activation_offloading": false, - "fine_grained_offloading_max_inflight_offloads": null, - "finetune": false, - "first_last_layers_bf16": false, - "flash_attention_version": null, - "flash_decode": false, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_dump_path": null, - "flight_recorder_extra_dump_on_exec": true, - "flight_recorder_include_only_active": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_trace_buffer_size": 2000, - "fp16": false, - "fp16_lm_cross_entropy": false, - "fp32_residual_connection": false, - "fp4": null, - "fp4_param_gather": false, - "fp4_quantizer_factory": null, - "fp4_recipe": "nvfp4", - "fp8": null, - "fp8_amax_compute_algo": "most_recent", - "fp8_amax_history_len": 1, - "fp8_interval": 1, - "fp8_margin": 0, - "fp8_output_proj": false, - "fp8_param_gather": false, - "fp8_quantizer_factory": null, - "fp8_recipe": "delayed", - "fp8_wgrad": true, - "freeze_all_layers": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "fsdp_double_buffer": false, - "fsdp_manual_registration": false, - "ft_num_warmup_iters": 5, - "full_validation": false, - "fused_residual_rmsnorm": false, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "gdp_num_householder": 3, - "global_batch_size": 128, - "glu_linear_offset": 0.0, - "gpu_sniff_test_interval": null, - "grad_reduce_in_bf16": false, - "gradient_accumulation_fusion": false, - "gradient_reduce_div_fusion": true, - "group_query_attention": false, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "grpo_group_size": 2, - "grpo_iterations": 2, - "grpo_kl_beta": 0.001, - "grpo_prompts_per_step": 32, - "gtp_expert_remat_nccl_ub": false, - "gtp_remat_nccl_ub": false, - "gtp_remat_opt_in_modules": "[]", - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "gtp_weight_remat_size": 1, - "head_lr_mult": 1.0, - "heterogeneous_layers_config_encoded_json": null, - "heterogeneous_layers_config_path": null, - "hfsdp_param_gather_overlap": false, - "hidden_dropout": 0.0, - "hidden_size": 1408, - "hierarchical_context_parallel_sizes": null, - "high_priority_a2a_comm_stream": false, - "high_priority_stream_groups": "[]", - "hybrid_context_parallel": false, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "hysteresis": 2, - "ict_head_size": null, - "ict_load": null, - "img_h": 224, - "img_w": 224, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "inference_batch_times_seqlen_threshold": -1, - "inference_coordinator_port": null, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_cuda_graph_scope": "InferenceCudaGraphScope.none", - "inference_disable_ep_consensus": false, - "inference_disable_triton_nvls_kernels": false, - "inference_dynamic_batching": false, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_sampling_backend": "torch", - "inference_dynamic_batching_track_generated_token_events": false, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_unified_memory_level": 0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_fuse_tp_communication": false, - "inference_grouped_gemm_backend": "vllm", - "inference_logging_step_interval": 0, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_moe_disable_fused_quant_kernels": false, - "inference_moe_token_dispatcher_type": "nvls", - "inference_rng_tracker": false, - "inference_shards": null, - "inference_text_gen_server_logging": false, - "inference_use_synchronous_zmq_collectives": false, - "inference_wandb_logging": false, - "init_method_std": 0.02, - "init_method_xavier_uniform": false, - "init_model_with_meta_device": false, - "initial_loss_scale": 4294967296, - "inprocess_active_world_size": 16, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_empty_cuda_cache": false, - "inprocess_granularity": "node", - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_interval": 30, - "inprocess_heartbeat_timeout": 60, - "inprocess_last_call_wait": 1, - "inprocess_max_iterations": null, - "inprocess_monitor_process_interval": 1.0, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_restart": false, - "inprocess_soft_timeout": 60, - "inprocess_termination_grace_time": 1, - "is_hybrid_model": false, - "iter_per_epoch": 1250, - "iteration": 0, - "iterations_to_skip": "[]", - "keep_fp8_transpose_cache": false, - "kitchen_attention_backend": "sdpa", - "kv_channels": 128, - "kv_lora_rank": 32, - "langrl_env_config": null, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "lazy_mpu_init": false, - "linear_attention_freq": null, - "linear_conv_kernel_dim": 4, - "linear_cp_layout": "zigzag", - "linear_cp_mode": "headwise", - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "linear_value_head_dim": 128, - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "load": null, - "load_main_params_from_ckpt": false, - "local_rank": 0, - "log_device_memory_used": false, - "log_energy": false, - "log_interval": 1, - "log_loss_scale_to_tensorboard": true, - "log_max_attention_logit": false, - "log_memory_interval": null, - "log_memory_to_tensorboard": false, - "log_num_zeros_in_grad": false, - "log_params_norm": false, - "log_progress": false, - "log_straggler": false, - "log_throughput": false, - "log_timers_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_world_size_to_tensorboard": false, - "logging_level": null, - "logit_dtype": null, - "logits_load_decode_threads": 4, - "logits_load_dir": null, - "logits_load_ignore_errors": false, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_msc_prefetch_depth": 2, - "logits_load_prefetch_factor": 3, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "loop_input_write": "ifm_diagonal", - "loss_scale": null, - "loss_scale_window": 1000, - "lr": 0.007470638533717786, - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_decay_style": "linear", - "lr_warmup_fraction": null, - "lr_warmup_init": 0.0, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_wsd_decay_style": "linear", - "main_grads_dtype": "torch.float32", - "main_params_dtype": "torch.float32", - "make_vocab_size_divisible_by": 64, - "mamba_head_dim": 64, - "mamba_inference_conv_states_dtype": "torch.bfloat16", - "mamba_inference_ssm_states_dtype": "torch.bfloat16", - "mamba_num_groups": 8, - "mamba_num_heads": null, - "mamba_state_dim": 128, - "mamba_training_ssm_states_dtype": null, - "manual_gc": false, - "manual_gc_eval": true, - "manual_gc_interval": 0, - "mask_factor": 1.0, - "mask_prob": 0.15, - "mask_type": "random", - "masked_softmax_fusion": true, - "max_position_embeddings": 2048, - "max_seqlen_per_dp_cp_rank": null, - "max_tokens_to_oom": 12000, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_main_params_dtype": "torch.float32", - "megatron_fsdp_max_pool_double_buffer": false, - "megatron_fsdp_version": 1, - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "memory_snapshot_path": "snapshot.pickle", - "merge_file": null, - "metadata_path": null, - "mhc_fused_backend": "auto", - "mhc_init_gating_factor": 0.01, - "mhc_num_residual_streams": 4, - "mhc_recompute_layer_num": null, - "mhc_sinkhorn_iterations": 20, - "micro_batch_size": 4, - "microbatch_group_size_per_vp_stage": null, - "mid_level_dataset_surplus": 0.005, - "min_loss_scale": 1.0, - "min_lr": 0.0007470638533717787, - "min_offloaded_tensor_size": 1048576, - "mla_down_proj_fusion": false, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "mmap_bin_files": true, - "mock_data": false, - "moe_apply_probs_on_input": false, - "moe_aux_loss_coeff": 0.0, - "moe_combine_bwd_dtype": "bf16", - "moe_deepep_num_sms": null, - "moe_dispatch_fwd_dtype": "bf16", - "moe_enable_deepep": false, - "moe_enable_routing_replay": false, - "moe_expert_capacity_factor": null, - "moe_expert_rank_capacity_factor": null, - "moe_ffn_hidden_size": null, - "moe_flex_dispatcher_backend": "deepep", - "moe_flex_dispatcher_num_sms": null, - "moe_grouped_gemm": false, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_input_jitter_eps": null, - "moe_latent_size": null, - "moe_layer_freq": 1, - "moe_layer_recompute": false, - "moe_mlp_glu_interleave_size": null, - "moe_ncclep_zero_copy": false, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_paged_stash": false, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_page_size": 64, - "moe_per_layer_logging": false, - "moe_permute_fusion": false, - "moe_permute_fusion_into_hybridep": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_dtype": null, - "moe_router_enable_expert_bias": false, - "moe_router_force_biased": null, - "moe_router_force_load_balancing": false, - "moe_router_fusion": false, - "moe_router_group_topk": null, - "moe_router_load_balancing_type": "aux_loss", - "moe_router_num_groups": null, - "moe_router_padding_for_fp8": false, - "moe_router_padding_for_quantization": false, - "moe_router_pre_softmax": false, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_score_function": "softmax", - "moe_router_skip_muon": false, - "moe_router_topk": 2, - "moe_router_topk_scaling_factor": null, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_dump_weights": false, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_path": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_overlap": false, - "moe_single_grouped_bias": false, - "moe_single_grouped_weight": false, - "moe_token_dispatcher_type": "allgather", - "moe_token_drop_policy": "probs", - "moe_upcycling_granularity": 1, - "moe_use_grouped_tensor": false, - "moe_use_norm_before_up_proj": false, - "moe_use_upcycling": false, - "moe_z_loss_coeff": null, - "mrope_section": null, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_num_layers": null, - "mtp_standalone": false, - "mtp_use_repeated_layer": false, - "multi_latent_attention": false, - "multiple_validation_sets": false, - "muon_coefficient_type": "quintic", - "muon_extra_scale_factor": 1.0, - "muon_fp32_matmul_prec": "medium", - "muon_momentum": 0.95, - "muon_nesterov": true, - "muon_num_ns_steps": 5, - "muon_scalar_optimizer": "adam", - "muon_scale_mode": "spectral", - "muon_split_qkv": true, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "mup_attn_scale_power": 1.0, - "mup_base_head_dim": null, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_width_mult": 1.0, - "nccl_all_reduce_for_prefill": false, - "nccl_communicator_config_path": null, - "nccl_ub": false, - "no_load_optim": null, - "no_load_rng": null, - "no_persist_layer_norm": false, - "no_rope_freq": null, - "no_save_optim": null, - "no_save_rng": null, - "no_weight_decay_cond_type": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "non_persistent_local_ckpt_dir": null, - "non_persistent_save_interval": null, - "normalization": "RMSNorm", - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "num_attention_heads": 11, - "num_channels": 3, - "num_classes": 1000, - "num_dataset_builder_threads": 1, - "num_distributed_optimizer_instances": 1, - "num_experts": null, - "num_floating_point_operations_so_far": 0, - "num_layers": 20, - "num_layers_at_end_in_bf16": 1, - "num_layers_at_start_in_bf16": 1, - "num_layers_per_virtual_pipeline_stage": null, - "num_query_groups": 11, - "num_speculative_tokens": 0, - "num_virtual_stages_per_pipeline_rank": null, - "num_workers": 0, - "nvtx_ranges": false, - "object_storage_cache_path": null, - "offload_modules": "[]", - "offset_sampling_seed_by_dp_rank": true, - "one_logger_async": false, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "online_data": true, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_dataset_subset": "sample-100BT", - "online_latency_log_dir": null, - "online_pack_sequences": false, - "online_prefetch_batches": 16, - "online_raw_manifest": null, - "online_shuffle_buffer_records": 128, - "online_source_mode": "stream", - "online_source_split": "train", - "online_split_seed": 260421106, - "online_text_column": "text", - "online_tokenizer_batch_size": 256, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_validation_fraction": 0.001, - "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json", - "onnx_safe": null, - "openai_gelu": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "outer_dp_sharding_strategy": "no_shard", - "output_bert_embeddings": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "overlap_grad_reduce": false, - "overlap_moe_expert_parallel_comm": false, - "overlap_p2p_comm": false, - "overlap_p2p_comm_warmup_flush": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "override_ckpt_iteration": null, - "override_opt_param_scheduler": false, - "pad_vocab_size": true, - "padded_vocab_size": 32064, - "paper_base_matrix_lr": 0.014, - "paper_embedding_lr": 0.3, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "paper_reference_batch_tokens": 262144, - "paper_reference_tokens": 983315200, - "paper_reference_width": 640, - "paper_transfer": "Paper2604Transfer(width=1408, total_tokens=7000031232, batch_tokens=262144, reference_width=640, reference_tokens=983315200, reference_batch_tokens=262144)", - "paper_unembedding_lr": 0.004, - "params_dtype": "torch.bfloat16", - "patch_dim": 16, - "per_dataset_sequences_path": null, - "per_split_data_args_path": null, - "perform_initialization": true, - "perform_rl_step": false, - "phase_transition_iterations": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_layout": null, - "pipeline_model_parallel_size": 1, - "position_embedding_type": "rope", - "pretrained_checkpoint": null, - "profile": false, - "profile_ranks": "[]", - "profile_step_end": 12, - "profile_step_start": 10, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "pytorch_profiler_collect_shapes": false, - "q_lora_rank": null, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "qk_head_dim": 128, - "qk_l2_norm": false, - "qk_layernorm": false, - "qk_pos_emb_head_dim": 64, - "query_in_block_prob": 0.1, - "quick_geglu": false, - "rampup_batch_size": null, - "rank": 0, - "recipe_optimizer": "{'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}", - "recipe_overrides": "{}", - "recipe_requested_batch_tokens": 262144, - "recipe_schedule": "{'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}", - "recompute_granularity": null, - "recompute_method": null, - "recompute_modules": null, - "recompute_num_layers": null, - "record_memory_history": false, - "record_shapes": false, - "refit_execution_batch_bytes": null, - "refit_method": "gloo", - "relative_attention_max_distance": 128, - "relative_attention_num_buckets": 32, - "replication": false, - "replication_factor": 2, - "replication_jump": null, - "rerun_mode": "validate_results", - "reset_attention_mask": false, - "reset_position_ids": false, - "result_rejected_tracker_filename": null, - "retriever_report_topk_accuracies": "[]", - "retriever_score_scaling": false, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "rl_consumption_granularity": "B", - "rl_default_temperature": 1.0, - "rl_default_top_k": -1, - "rl_default_top_p": 0, - "rl_durable_rollout_bank": false, - "rl_generation_lag": null, - "rl_importance_sampling_truncation_coef": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_logprobs_is_correction": false, - "rl_inference_model_unified_memory_level": 0, - "rl_inference_parsers": "[]", - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_tensor_model_parallel_size": null, - "rl_kv_cache_management_mode": "persist", - "rl_max_inflight_requests": null, - "rl_offload_inference_model_weights_when_idle": false, - "rl_offload_optimizer_during_inference": false, - "rl_partial_rollouts": false, - "rl_persist_cuda_graphs": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_prompts_per_eval": 32, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "rl_sequence_packing_algo": "fifo", - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_skip_bos_token": false, - "rl_submission_granularity": "B", - "rl_training_cuda_graphs": false, - "rl_use_sequence_packing": false, - "rl_verify_model_weights_swap": false, - "rope_scaling_factor": 8.0, - "rope_type": null, - "rotary_base": 10000, - "rotary_interleaved": false, - "rotary_percent": 1.0, - "rotary_scaling_factor": 1.0, - "rotary_seq_len_interpolation_factor": null, - "run_workload_inspector_server": false, - "sample_rate": 1.0, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/checkpoints", - "save_activations_interval": null, - "save_dgrads_interval": null, - "save_interval": 4000, - "save_params_interval": null, - "save_retain_interval": null, - "save_tokenizer_assets": true, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "seed": 42, - "seq_length": 2048, - "sequence_packing_scheduler": null, - "sequence_parallel": false, - "sft": false, - "sft_mock_dataset_config_json": null, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sgd_momentum": 0.9, - "sharp_enabled_group": null, - "short_seq_prob": 0.1, - "skip_train": false, - "skipped_train_samples": 0, - "softmax_type": "vanilla", - "spec": null, - "special_tokens": null, - "split": null, - "squared_relu": false, - "start_eval_at_iter": null, - "start_weight_decay": 0.0, - "step_batch_size_schedule": null, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "strict_fsdp_dtensor_load": true, - "suggested_communication_unit_size": null, - "swiglu": false, - "swin_backbone_type": "tiny", - "symmetric_ar_type": null, - "te_precision_config_file": null, - "te_rng_tracker": false, - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "test_data_path": null, - "test_mode": false, - "tiktoken_num_special_tokens": 1000, - "tiktoken_pattern": null, - "timing_log_level": 0, - "timing_log_option": "minmax", - "titles_data_path": null, - "tokenizer_hf_no_include_special_tokens": false, - "tokenizer_hf_no_use_fast": false, - "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_type": "HuggingFaceTokenizer", - "torch_fsdp2_reshard_after_forward": true, - "tp_comm_bootstrap_backend": "nccl", - "tp_comm_bulk_dgrad": true, - "tp_comm_bulk_wgrad": true, - "tp_comm_overlap": false, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_cfg": null, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "train_data_path": null, - "train_iters": 26703, - "train_samples": null, - "train_sync_interval": null, - "training_recipe": "paper2604_21106", - "training_recipe_metadata": "{'recipe': 'paper2604_21106', 'width': 1408, 'total_tokens': 7000031232, 'batch_tokens': 262144, 'reference_width': 640, 'reference_tokens': 983315200, 'reference_batch_tokens': 262144, 'base_lrs': {'matrix': 0.014, 'embedding': 0.3, 'unembedding': 0.004, 'norm': 0.005, 'other': 0.005}, 'peak_lrs': {'matrix': 0.007470638533717786, 'embedding': 0.2022599587389726, 'unembedding': 0.0026967994498529684, 'norm': 0.00337099931231621, 'other': 0.00337099931231621}, 'reference': 'paper vanilla 0-20-0 at width 640; configurable token anchor', 'tied_embedding_policy': 'unembedding (one AdamW update)', 'other_parameter_policy': 'norm LR (extension for non-paper scalars/biases)', 'weight_decay': 0.0, 'warmup_steps': 0, 'final_lr_fraction': 0.1, 'optimizer': {'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}, 'schedule': {'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}, 'group_weight_decay': {'matrix': 0.0, 'embedding': 0.0, 'unembedding': 0.0, 'norm': 0.0, 'other': 0.0}, 'recipe_overrides': {}, 'requested_batch_tokens': 262144, 'sequence_length': 2048, 'batch_assumptions': {'global_batch_tokens': 262144, 'sequence_length': 2048, 'round_up': False}, 'lr_transfer': 'mup_hyperp_released_source', 'assumptions': [], 'backend_kernel_assumptions': {}, 'paper_settings_overridden': False, 'model': {'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 651401344, 'feedback_parameters': 0, 'loop_input_write_parameters': 1985280, 'loop_adapter_parameters': 83271936}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'full_bandwidth_loop', 'objective': 'depth_loop_causal_lm', 'feedback_enabled': False, 'train_passes': 1, 'eval_passes': 1, 'feedback_pass_mixture': {1: 1.0}, 'feedback_progressive_stages': [], 'depth_num_loops': 2, 'logical_layers_per_pass': 40, 'shared_layer_weights': True, 'depth_share_kv': False, 'depth_kv_policy': 'recompute_each_loop', 'norm_placement': 'embedding once; recur/final after loop 2 before three refiners; no post-refiner norm', 'loop_input_write': 'ifm_diagonal', 'loop_input_write_source': 'raw_token_embeddings', 'loop_input_write_placement': 'before_every_depth_loop_after_initial_norm_emb', 'loop_input_write_sharing': 'all_depth_loops_and_outer_feedback_passes', 'loop_input_write_init': 'paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)', 'loop_final_adapter': 'lookahead_refiner', 'loop_adapter': {'ffn_size': 5632, 'state_input': 'second-loop hidden states after norm_recur and norm_final', 'after_depth_loop': 2, 'applications_per_pass': 1, 'terminal_norms': True, 'attention_backend': 'sdpa_auto', 'checkpoint_refiner': False, 'bank_layers': [0, 10, 19], 'bank_source': 'first depth loop, fresh on each feedback pass', 'weight_sharing': 'one adapter shared across feedback passes'}, 'block_application_multiplier_per_pass': 2.15, 'block_application_multiplier': 2.15}}", - "transformer_impl": "local", - "transformer_pipeline_model_parallel_size": 1, - "trust_remote_code": false, - "untie_embeddings_and_output_weights": true, - "use_checkpoint_args": false, - "use_checkpoint_opt_param_scheduler": false, - "use_cpu_initialization": null, - "use_dist_ckpt": false, - "use_dist_ckpt_deprecated": false, - "use_distributed_optimizer": false, - "use_flash_attn": false, - "use_fused_mhc": false, - "use_fused_weighted_squared_relu": false, - "use_gigatoken": false, - "use_gloo_process_groups": true, - "use_grouped_gemm_for_dense_mlp": false, - "use_grouped_gemm_for_shared_expert": false, - "use_kitchen_attention": false, - "use_layer_wise_distributed_optimizer": false, - "use_layer_wise_param_layout": true, - "use_legacy_static_engine": false, - "use_mamba_mem_eff_path": true, - "use_megatron_fsdp": false, - "use_mp_args_from_checkpoint_args": false, - "use_mup": false, - "use_one_sent_docs": false, - "use_persistent_ckpt_worker": false, - "use_precision_aware_optimizer": false, - "use_pytorch_profiler": false, - "use_ring_exchange_p2p": false, - "use_rope_scaling": false, - "use_rotary_position_embeddings": false, - "use_sharp": false, - "use_te_activation_func": false, - "use_tokenizer_model_from_checkpoint_args": true, - "use_torch_fsdp2": false, - "use_torch_optimizer_for_cpu_offload": false, - "use_tp_pp_dp_mapping": false, - "use_transformer_engine_op_fuser": false, - "use_varlen_dataset": false, - "v_head_dim": 128, - "valid_data_path": null, - "validation_set_names": null, - "variable_seq_lengths": false, - "varlen_mock_dataset_config_json": null, - "varlen_sbhd_validation": false, - "verify_integrity": false, - "virtual_pipeline_model_parallel_size": null, - "vision_backbone_type": "vit", - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vocab_extra_ids": 0, - "vocab_file": null, - "vocab_size": 32001, - "wandb_entity": "LoopTF-4-CSPs", - "wandb_exp_name": "paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM", - "wandb_project": "loopTF-workspace", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/wandb", - "weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "wgrad_deferral_limit": 0, - "window_attn_skip_freq": null, - "window_size": null, - "world_size": 16, - "yaml_cfg": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "yarn_original_max_position_embeddings": null - }, - "native_reconstruction": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "context_extension": "native RoPE, no scaling; at most 2x training length", - "max_supported_length": 4096, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 2, - "depth_share_kv": false, - "feedback_layers": [ - 20 - ], - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "loop_input_write": "ifm_diagonal", - "num_passes": 1 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 26703, - "training_context_length": 2048 - }, - "parameter_counts": { - "feedback_parameters": 0, - "loop_adapter_parameters": 83271936, - "loop_input_write_parameters": 1985280, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 651401344 - }, - "pipeline_model_parallel_size": 1, - "run_name": "paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM", - "tensor_model_parallel_size": 1, - "tokenizer": { - "added_tokens": [ - "<|assistant_end|>" - ], - "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "training_budget_tokens": 7000031232, - "training_code_revision": "82c473685b887bdf49b391763048ac06e0e3f6a3", - "training_tokens": 7000031232 -} diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env deleted file mode 100644 index d61d26b146696bdf770d8b1c998384132de18939..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env +++ /dev/null @@ -1,117 +0,0 @@ -# Vanilla shared-stack depth loop, L=2, with FBW completely disabled (K=1). -# Matched to 476M_7b_FBW0_K3_L2.env; see NO_FBW_BASELINES.md. -# Backbone: 475,851,904 non-embedding; with IFM and adapter: 651,401,344 total trainable parameters. -# Inject projected raw token embeddings before every loop; compute fresh KV each loop. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=2 -DEPTH_SHARE_KV=false -LOOP_INPUT_WRITE=ifm_diagonal -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -AUX_MTP_HORIZON=1 -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# One outer pass: ordinary next-token CE after the final depth loop. -# No feedback gate, shifted states, intermediate-loop loss, or gradient detach. -# 20 physical layers x 2 loops = 40 backbone + 3 final refiner applications. -# Normalize after the final depth loop, then run the adapter once before the head. -FEEDBACK_PASS_MIXTURE=1:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=1 -FEEDBACK_LOSS_WEIGHT=0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop_476m_7b_no_fbw_ADAPTER_IFM -WANDB_TAGS=product,loop,loop2,paper2604,476m,7b,no_fbw,K1,L2,ifm,input_injection,fresh_kv,lookahead_refiner -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 - -# Pin unpacked data and the full horizon against inherited launch settings. -ONLINE_PACK_SEQUENCES=false -EXIT_INTERVAL= diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json deleted file mode 100644 index 9496a103c316a56c9e6572321ab747fe4ada27cd..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "sha256": "ea6e017150ab3f8fb8347d5d430d42897b1c966d5ca3a3169d098a9ee5ca4f98", - "size": 517 - }, - "config.json": { - "sha256": "adfb06568fae7e00108783fab95cb768aedb04860644d19a7389240e733431bd", - "size": 52187 - }, - "launch_config.env": { - "sha256": "047976485fab887985c74af3b599189dccdf5caed056ce9ff6ad9c42b235eb8c", - "size": 4086 - }, - "model.safetensors": { - "sha256": "a3ebb5505dbd7ac0421d4dd182bee6b359f0d8bf04063aee0a90deb49f520ab6", - "size": 1302958576 - }, - "training_recipe.json": { - "sha256": "ca88d58bfcfe84db4bb1cff32fd7d3e3960d75623d174a73002dae7bf92b37f4", - "size": 5132 - } - }, - "note": "manifest.json does not hash itself" -} diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors deleted file mode 100644 index ee2630227aa7d6486c94cd427fdae35fd4b871c3..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a3ebb5505dbd7ac0421d4dd182bee6b359f0d8bf04063aee0a90deb49f520ab6 -size 1302958576 diff --git a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json b/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json deleted file mode 100644 index 0f747b0a0032fecaecd57fa693f3624b6fb693cc..0000000000000000000000000000000000000000 --- a/paper2604_loop2_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 651401344, - "feedback_parameters": 0, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "depth_loop_causal_lm", - "feedback_enabled": false, - "train_passes": 1, - "eval_passes": 1, - "feedback_pass_mixture": { - "1": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 2, - "logical_layers_per_pass": 40, - "shared_layer_weights": true, - "depth_share_kv": false, - "depth_kv_policy": "recompute_each_loop", - "norm_placement": "embedding once; recur/final after loop 2 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "second-loop hidden states after norm_recur and norm_final", - "after_depth_loop": 2, - "applications_per_pass": 1, - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 2.15, - "block_application_multiplier": 2.15 - } -} diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md deleted file mode 100644 index 686b59f75f199fbfb89bcda562081b72b115a3b0..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# L2 Adapter IFM final weights - -Final step 26,703 native weights-only export for the paper2604 L2 loop with IFM diagonal input write and terminal Look-ahead adapter. - -`model.safetensors` is a native Megatron state dict, not a Transformers AutoModel checkpoint. It excludes optimizer, scheduler, RNG, dataloader, logs, datasets, and credentials. Use the recorded full_bandwidth_loop configuration and strict loading; `config.json` records native reconstruction metadata. - -Use `manifest.json` to verify payload hashes. diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json deleted file mode 100644 index ce24b72cc0094d81464684f2af1cb319b17d7521..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/config.json +++ /dev/null @@ -1,1117 +0,0 @@ -{ - "architecture": "full_bandwidth_loop", - "checkpoint_format": "safetensors_native_megatron_state_dict", - "checkpoint_version": 3.0, - "excluded_null_extra_state_keys": [ - "decoder.stack.layers.0.self_attention.linear_proj._extra_state", - "decoder.stack.layers.0.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.0.mlp.linear_fc1._extra_state", - "decoder.stack.layers.0.mlp.linear_fc2._extra_state", - "decoder.stack.layers.1.self_attention.linear_proj._extra_state", - "decoder.stack.layers.1.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.1.mlp.linear_fc1._extra_state", - "decoder.stack.layers.1.mlp.linear_fc2._extra_state", - "decoder.stack.layers.2.self_attention.linear_proj._extra_state", - "decoder.stack.layers.2.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.2.mlp.linear_fc1._extra_state", - "decoder.stack.layers.2.mlp.linear_fc2._extra_state", - "decoder.stack.layers.3.self_attention.linear_proj._extra_state", - "decoder.stack.layers.3.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.3.mlp.linear_fc1._extra_state", - "decoder.stack.layers.3.mlp.linear_fc2._extra_state", - "decoder.stack.layers.4.self_attention.linear_proj._extra_state", - "decoder.stack.layers.4.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.4.mlp.linear_fc1._extra_state", - "decoder.stack.layers.4.mlp.linear_fc2._extra_state", - "decoder.stack.layers.5.self_attention.linear_proj._extra_state", - "decoder.stack.layers.5.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.5.mlp.linear_fc1._extra_state", - "decoder.stack.layers.5.mlp.linear_fc2._extra_state", - "decoder.stack.layers.6.self_attention.linear_proj._extra_state", - "decoder.stack.layers.6.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.6.mlp.linear_fc1._extra_state", - "decoder.stack.layers.6.mlp.linear_fc2._extra_state", - "decoder.stack.layers.7.self_attention.linear_proj._extra_state", - "decoder.stack.layers.7.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.7.mlp.linear_fc1._extra_state", - "decoder.stack.layers.7.mlp.linear_fc2._extra_state", - "decoder.stack.layers.8.self_attention.linear_proj._extra_state", - "decoder.stack.layers.8.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.8.mlp.linear_fc1._extra_state", - "decoder.stack.layers.8.mlp.linear_fc2._extra_state", - "decoder.stack.layers.9.self_attention.linear_proj._extra_state", - "decoder.stack.layers.9.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.9.mlp.linear_fc1._extra_state", - "decoder.stack.layers.9.mlp.linear_fc2._extra_state", - "decoder.stack.layers.10.self_attention.linear_proj._extra_state", - "decoder.stack.layers.10.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.10.mlp.linear_fc1._extra_state", - "decoder.stack.layers.10.mlp.linear_fc2._extra_state", - "decoder.stack.layers.11.self_attention.linear_proj._extra_state", - "decoder.stack.layers.11.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.11.mlp.linear_fc1._extra_state", - "decoder.stack.layers.11.mlp.linear_fc2._extra_state", - "decoder.stack.layers.12.self_attention.linear_proj._extra_state", - "decoder.stack.layers.12.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.12.mlp.linear_fc1._extra_state", - "decoder.stack.layers.12.mlp.linear_fc2._extra_state", - "decoder.stack.layers.13.self_attention.linear_proj._extra_state", - "decoder.stack.layers.13.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.13.mlp.linear_fc1._extra_state", - "decoder.stack.layers.13.mlp.linear_fc2._extra_state", - "decoder.stack.layers.14.self_attention.linear_proj._extra_state", - "decoder.stack.layers.14.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.14.mlp.linear_fc1._extra_state", - "decoder.stack.layers.14.mlp.linear_fc2._extra_state", - "decoder.stack.layers.15.self_attention.linear_proj._extra_state", - "decoder.stack.layers.15.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.15.mlp.linear_fc1._extra_state", - "decoder.stack.layers.15.mlp.linear_fc2._extra_state", - "decoder.stack.layers.16.self_attention.linear_proj._extra_state", - "decoder.stack.layers.16.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.16.mlp.linear_fc1._extra_state", - "decoder.stack.layers.16.mlp.linear_fc2._extra_state", - "decoder.stack.layers.17.self_attention.linear_proj._extra_state", - "decoder.stack.layers.17.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.17.mlp.linear_fc1._extra_state", - "decoder.stack.layers.17.mlp.linear_fc2._extra_state", - "decoder.stack.layers.18.self_attention.linear_proj._extra_state", - "decoder.stack.layers.18.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.18.mlp.linear_fc1._extra_state", - "decoder.stack.layers.18.mlp.linear_fc2._extra_state", - "decoder.stack.layers.19.self_attention.linear_proj._extra_state", - "decoder.stack.layers.19.self_attention.linear_qkv._extra_state", - "decoder.stack.layers.19.mlp.linear_fc1._extra_state", - "decoder.stack.layers.19.mlp.linear_fc2._extra_state", - "output_layer._extra_state" - ], - "excluded_state": [ - "optimizer", - "opt_param_scheduler", - "rng_state", - "dataloader" - ], - "iteration": 26703, - "loading": { - "format": "native Megatron model state; not a Transformers AutoModel", - "instructions": "Use loop_transformer_backends.benchmarks.models.build_native on the original checkpoint schema, or reconstruct the recorded full_bandwidth_loop/IFM/adapter architecture before strict state loading." - }, - "model_arguments": { - "_is_global_batch_size_explicitly_specified": true, - "account_for_embedding_in_pipeline_split": false, - "account_for_loss_in_pipeline_split": false, - "accumulate_allreduce_grads_in_fp32": true, - "activation_func_clamp_value": null, - "activation_offload_fraction": 1.0, - "adam_beta1": 0.8, - "adam_beta2": 0.95, - "adam_eps": 1e-10, - "adaptive_muon_beta2": 0.95, - "adaptive_muon_eps": 1e-10, - "adaptive_muon_moment2_method": "normuon", - "add_bias_linear": false, - "add_position_embedding": true, - "add_qkv_bias": false, - "adlr_autoresume": false, - "adlr_autoresume_interval": 1000, - "align_grad_reduce": true, - "align_param_gather": false, - "allow_ambiguous_pad_tokens": false, - "app_tag_run_name": null, - "app_tag_run_version": "0.0.0", - "apply_query_key_layer_scaling": false, - "apply_residual_connection_post_layernorm": false, - "apply_rope_fusion": false, - "apply_wd_to_qk_layernorm": false, - "architecture": "full_bandwidth_loop", - "architecture_module": "[]", - "async_ckpt_cpu_priority": 10, - "async_ckpt_io_priority": 3, - "async_ckpt_use_cpu_shm": false, - "async_save": false, - "async_strategy": "mcore", - "attention_backend": "AttnBackend.auto", - "attention_cp_layout": "zigzag", - "attention_dropout": 0.0, - "attention_output_gate": false, - "attention_softmax_in_fp32": false, - "auto_detect_ckpt_format": false, - "aux_mtp_horizon": 1, - "aux_mtp_loss_weight": 0.3, - "aux_mtp_rank": 128, - "backbone_metadata": "{'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 651401344, 'feedback_parameters': 0, 'loop_input_write_parameters': 1985280, 'loop_adapter_parameters': 83271936}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'full_bandwidth_loop', 'objective': 'depth_loop_causal_lm', 'feedback_enabled': False, 'train_passes': 1, 'eval_passes': 1, 'feedback_pass_mixture': {1: 1.0}, 'feedback_progressive_stages': [], 'depth_num_loops': 6, 'logical_layers_per_pass': 120, 'shared_layer_weights': True, 'depth_share_kv': False, 'depth_kv_policy': 'recompute_each_loop', 'norm_placement': 'embedding once; recur/final after loop 6 before three refiners; no post-refiner norm', 'loop_input_write': 'ifm_diagonal', 'loop_input_write_source': 'raw_token_embeddings', 'loop_input_write_placement': 'before_every_depth_loop_after_initial_norm_emb', 'loop_input_write_sharing': 'all_depth_loops_and_outer_feedback_passes', 'loop_input_write_init': 'paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)', 'loop_final_adapter': 'lookahead_refiner', 'loop_adapter': {'ffn_size': 5632, 'state_input': 'final-loop hidden states after norm_recur and norm_final', 'after_depth_loop': 6, 'applications_per_pass': 1, 'terminal_norms': True, 'attention_backend': 'sdpa_auto', 'checkpoint_refiner': False, 'bank_layers': [0, 10, 19], 'bank_source': 'first depth loop, fresh on each feedback pass', 'weight_sharing': 'one adapter shared across feedback passes'}, 'block_application_multiplier_per_pass': 6.15, 'block_application_multiplier': 6.15}", - "backbone_profile": "paper2604_vanilla", - "barrier_with_L1_time": true, - "batch_invariant_backend": "te_native", - "batch_invariant_collective": "ordered", - "batch_invariant_mode": false, - "bert_binary_head": true, - "bert_embedder_type": "megatron", - "bert_load": null, - "bf16": true, - "bias_dropout_fusion": false, - "bias_gelu_fusion": false, - "bias_swiglu_fusion": true, - "biencoder_projection_dim": 0, - "biencoder_shared_query_context_model": false, - "block_data_path": null, - "cache_mla_latents": false, - "calc_ft_timeouts": false, - "calculate_per_token_loss": false, - "chat_template": null, - "check_for_large_grads": false, - "check_for_nan_in_loss_and_grad": true, - "check_for_spiky_loss": false, - "check_weight_hash_across_dp_replicas_interval": null, - "ckpt_assume_constant_structure": false, - "ckpt_convert_format": null, - "ckpt_convert_save": null, - "ckpt_convert_update_legacy_dist_opt_format": false, - "ckpt_drop_redundant_extra_state": false, - "ckpt_format": "torch", - "ckpt_fully_parallel_load": false, - "ckpt_fully_parallel_load_exchange_algo": "broadcast", - "ckpt_fully_parallel_load_per_rank_objects": false, - "ckpt_fully_parallel_load_process_group": "dp", - "ckpt_fully_parallel_save": true, - "ckpt_fully_parallel_save_deprecated": false, - "ckpt_fully_parallel_save_process_group": "dp", - "ckpt_load_validate_sharding_integrity": true, - "ckpt_pg_tensors_cache_create": false, - "ckpt_pg_tensors_cache_path": null, - "ckpt_step": null, - "classes_fraction": 1.0, - "clip_grad": 0.0, - "clone_scatter_output_in_embedding": true, - "config_logger_dir": "", - "consumed_train_samples": 3417984, - "consumed_valid_samples": 68608, - "context_parallel_size": 1, - "cp_comm_type": "['p2p']", - "cpu_offloading_num_layers": 0, - "cpu_offloading_retain_pinned_cpu_buffers": false, - "create_all_gather_group": false, - "create_attention_mask_in_dataloader": false, - "cross_entropy_fusion_impl": "native", - "cross_entropy_loss_fusion": false, - "csa_compress_ratios": null, - "csa_compress_rotary_base": 40000.0, - "csa_dense_mode": false, - "csa_window_size": 128, - "cuda_graph_impl": "none", - "cuda_graph_modules": "[]", - "cuda_graph_warmup_steps": 3, - "curr_iteration": 26702, - "data_args_path": null, - "data_cache_path": null, - "data_parallel_random_init": false, - "data_parallel_sharding_strategy": "optim_grads_params", - "data_parallel_size": 16, - "data_path": null, - "data_per_class_fraction": 1.0, - "data_sharding": true, - "dataloader_defer_npy_index_mmap": false, - "dataloader_fast_cache_load": false, - "dataloader_inter_document_masking": false, - "dataloader_save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/dataloader", - "dataloader_type": "external", - "ddp_average_in_collective": false, - "ddp_bucket_size": null, - "ddp_num_buckets": null, - "ddp_pad_buckets_for_high_nccl_busbw": false, - "ddp_param_name_patterns_for_fp32_local_accumulation": "[]", - "ddp_reduce_scatter_with_fp32_accumulation": false, - "decode_only_cuda_graphs": false, - "decoder_first_pipeline_num_layers": null, - "decoder_last_pipeline_num_layers": null, - "decoder_num_layers": null, - "decoder_seq_length": null, - "decoupled_lr": null, - "decoupled_min_lr": null, - "decrease_batch_size_if_needed": false, - "defer_embedding_wgrad_compute": false, - "delay_offload_until_cuda_graph": false, - "delay_wgrad_compute": false, - "delta_offload_bytes_across_pp_ranks": 0, - "deprecated_use_mcore_models": false, - "depth_num_loops": 6, - "depth_share_kv": false, - "deterministic_mode": false, - "dino_bottleneck_size": 256, - "dino_freeze_last_layer": 1, - "dino_head_hidden_size": 2048, - "dino_local_crops_number": 10, - "dino_local_img_size": 96, - "dino_norm_last_layer": false, - "dino_teacher_temp": 0.07, - "dino_warmup_teacher_temp": 0.04, - "dino_warmup_teacher_temp_epochs": 30, - "disable_bf16_reduced_precision_matmul": false, - "disable_jit_fuser": false, - "disable_msc_deprecated": false, - "disable_straggler_on_startup": false, - "disable_symmetric_registration": false, - "dist_ckpt_format_deprecated": null, - "dist_ckpt_optim_fully_reshardable": false, - "dist_ckpt_save_pre_mcore_014": false, - "dist_ckpt_strictness": "assume_ok_unexpected", - "dist_ckpt_workers": 1, - "distrib_optim_fully_reshardable_mem_efficient": false, - "distribute_saved_activations": false, - "distributed_backend": "nccl", - "distributed_timeout_minutes": 10, - "distributed_timeout_seconds_after_init": null, - "do_test": 0, - "do_train": 1, - "do_valid": 1, - "dsa_indexer_head_dim": null, - "dsa_indexer_k_norm_epsilon": null, - "dsa_indexer_k_norm_fp32": false, - "dsa_indexer_loss_coeff": null, - "dsa_indexer_n_heads": null, - "dsa_indexer_rope_interleaved": false, - "dsa_indexer_rotate_activation": true, - "dsa_indexer_scoring_relu": true, - "dsa_indexer_skip_topk_offset": 0, - "dsa_indexer_topk": null, - "dsa_indexer_topk_freq": 1, - "dsa_indexer_use_sparse_loss": false, - "dsa_kernel_backend": "none", - "dump_param_to_param_group_map": null, - "embedding_init_method_std": null, - "embedding_path": null, - "empty_unused_memory_level": 0, - "enable_chunked_prefill": false, - "enable_cuda_graph": false, - "enable_experimental": false, - "enable_ft_package": false, - "enable_full_sharding_in_hsdp": false, - "enable_mhc_connections": false, - "enable_msc": false, - "enable_one_logger": true, - "encoder_num_layers": 20, - "encoder_seq_length": 2048, - "end_weight_decay": 0.0, - "eod_mask_loss": false, - "ep_overlap_early_attn_memory_release": false, - "error_injection_rate": 0, - "error_injection_type": "transient_error", - "eval_at_start": true, - "eval_global_batch_size": 128, - "eval_interval": 100, - "eval_iters": 2, - "eval_micro_batch_size": 4, - "evaluation_metadata": "{'eval_at_start': True, 'initial_iters': 2, 'intermediate_iters': 2, 'final_validation_iters': 270, 'test_iters': 0, 'global_batch_size': 128, 'sequence_length': 2048, 'intermediate_input_tokens': 524288, 'final_validation_sequences': 34560, 'final_validation_input_tokens': 70778880, 'validation_iterator': 'continues_existing_held_out_iterator'}", - "evidence_data_path": null, - "exit_duration_in_mins": null, - "exit_interval": null, - "exit_on_missing_checkpoint": true, - "exit_signal": 15, - "exit_signal_handler": false, - "exit_signal_handler_for_dataloader": false, - "exit_signal_handler_for_training": false, - "exp_avg_dtype": "torch.float32", - "exp_avg_sq_dtype": "torch.float32", - "experimental_attention_variant": null, - "expert_data_parallel_sharding_strategy": null, - "expert_gtp_weight_remat_size": 1, - "expert_model_parallel_size": 1, - "expert_tensor_parallel_num_weight_shards": 1, - "expert_tensor_parallel_size": 1, - "external_cuda_graph": false, - "fake_process_group": false, - "fault_injector_delay_start_iteration": null, - "fault_injector_fault_delay": null, - "fault_injector_fault_probabilities": null, - "fault_injector_fault_types": null, - "fault_injector_mtti_seconds": null, - "fault_injector_num_ranks": null, - "fault_injector_offset_seconds": null, - "fault_injector_ranks": null, - "fault_injector_seed": null, - "feedback_eval_passes": 1, - "feedback_jitter_sigma": 0.0, - "feedback_layers": null, - "feedback_log_hidden_change": false, - "feedback_loss_weight": 0.0, - "feedback_normuon_beta2": 0.95, - "feedback_pass_mixture": "1:1.0", - "feedback_prefix_mixin": false, - "feedback_progressive_stages": "[]", - "feedback_scalar_lr": 0.0005, - "feedback_scalar_weight_decay": 0.0, - "feedback_z_loss_coefficient": 0.0, - "feedback_z_loss_start_fraction": 1.0, - "ffn_hidden_size": 5632, - "fim_data": false, - "fim_eod_token": "<|endoftext|>", - "fim_fragment_rate": null, - "fim_middle_token": "", - "fim_no_prefix": null, - "fim_pad_token": "", - "fim_prefix_token": "", - "fim_rate": 0.5, - "fim_split_sample": null, - "fim_spm_rate": 0.5, - "fim_suffix_token": "", - "final_eval_iters": 270, - "fine_grained_activation_offloading": false, - "fine_grained_offloading_max_inflight_offloads": null, - "finetune": false, - "first_last_layers_bf16": false, - "flash_attention_version": null, - "flash_decode": false, - "flight_recorder_dump_on_timeout": true, - "flight_recorder_dump_path": null, - "flight_recorder_extra_dump_on_exec": true, - "flight_recorder_include_only_active": true, - "flight_recorder_include_stack_trace": false, - "flight_recorder_trace_buffer_size": 2000, - "fp16": false, - "fp16_lm_cross_entropy": false, - "fp32_residual_connection": false, - "fp4": null, - "fp4_param_gather": false, - "fp4_quantizer_factory": null, - "fp4_recipe": "nvfp4", - "fp8": null, - "fp8_amax_compute_algo": "most_recent", - "fp8_amax_history_len": 1, - "fp8_interval": 1, - "fp8_margin": 0, - "fp8_output_proj": false, - "fp8_param_gather": false, - "fp8_quantizer_factory": null, - "fp8_recipe": "delayed", - "fp8_wgrad": true, - "freeze_all_layers": false, - "fsdp_db_use_persist_buf_on_alloc_fail": false, - "fsdp_double_buffer": false, - "fsdp_manual_registration": false, - "ft_num_warmup_iters": 5, - "full_validation": false, - "fused_residual_rmsnorm": false, - "gdp_cutedsl_kernel": false, - "gdp_num_chunk_states_to_recompute": 2, - "gdp_num_householder": 3, - "global_batch_size": 128, - "glu_linear_offset": 0.0, - "gpu_sniff_test_interval": null, - "grad_reduce_in_bf16": false, - "gradient_accumulation_fusion": false, - "gradient_reduce_div_fusion": true, - "group_query_attention": false, - "grpo_clamp_eps_lower": 0.01, - "grpo_clamp_eps_upper": 0.01, - "grpo_entropy_term_weight": 0.0, - "grpo_filter_groups_with_same_reward": false, - "grpo_group_size": 2, - "grpo_iterations": 2, - "grpo_kl_beta": 0.001, - "grpo_prompts_per_step": 32, - "gtp_expert_remat_nccl_ub": false, - "gtp_remat_nccl_ub": false, - "gtp_remat_opt_in_modules": "[]", - "gtp_remat_reduce_scatter_with_fp32_accumulation": false, - "gtp_weight_remat_size": 1, - "head_lr_mult": 1.0, - "heterogeneous_layers_config_encoded_json": null, - "heterogeneous_layers_config_path": null, - "hfsdp_param_gather_overlap": false, - "hidden_dropout": 0.0, - "hidden_size": 1408, - "hierarchical_context_parallel_sizes": null, - "high_priority_a2a_comm_stream": false, - "high_priority_stream_groups": "[]", - "hybrid_context_parallel": false, - "hybrid_layer_pattern": null, - "hybrid_override_pattern": null, - "hysteresis": 2, - "ict_head_size": null, - "ict_load": null, - "img_h": 224, - "img_w": 224, - "indexer_batch_size": 128, - "indexer_log_interval": 1000, - "inference_batch_times_seqlen_threshold": -1, - "inference_coordinator_port": null, - "inference_cuda_graph_all_prefills": false, - "inference_cuda_graph_max_tokens": 512, - "inference_cuda_graph_scope": "InferenceCudaGraphScope.none", - "inference_disable_ep_consensus": false, - "inference_disable_triton_nvls_kernels": false, - "inference_dynamic_batching": false, - "inference_dynamic_batching_allow_stale_multimodal_embeddings": false, - "inference_dynamic_batching_async_sched_mode": "async", - "inference_dynamic_batching_block_size": 256, - "inference_dynamic_batching_buffer_size_gb": 40.0, - "inference_dynamic_batching_cuda_graph_mixed_prefill_count": 16, - "inference_dynamic_batching_cuda_graph_sizing_distribution": "hybrid", - "inference_dynamic_batching_enable_prefix_caching": false, - "inference_dynamic_batching_logprobs_mode": "raw_logprobs", - "inference_dynamic_batching_mamba_memory_ratio": null, - "inference_dynamic_batching_max_requests": null, - "inference_dynamic_batching_max_tokens": null, - "inference_dynamic_batching_media_cache_coordinator_policy": "affinity", - "inference_dynamic_batching_media_cache_routing_weight": 1.0, - "inference_dynamic_batching_num_cuda_graphs": 16, - "inference_dynamic_batching_paused_buffer_size_gb": null, - "inference_dynamic_batching_prefix_cache_ttl_seconds": 300.0, - "inference_dynamic_batching_prefix_caching_coordinator_policy": "longest_prefix", - "inference_dynamic_batching_prefix_caching_eviction_policy": "lru", - "inference_dynamic_batching_prefix_caching_mamba_gb": null, - "inference_dynamic_batching_prefix_caching_routing_alpha": 1.0, - "inference_dynamic_batching_sampling_backend": "torch", - "inference_dynamic_batching_track_generated_token_events": false, - "inference_dynamic_batching_track_paused_request_events": false, - "inference_dynamic_batching_unified_memory_level": 0, - "inference_dynamic_batching_vision_embedding_cache_max_bytes": 0, - "inference_flashinfer_mxfp8_token_capacity": null, - "inference_fuse_tp_communication": false, - "inference_grouped_gemm_backend": "vllm", - "inference_logging_step_interval": 0, - "inference_max_requests": 8, - "inference_max_seq_length": 2560, - "inference_moe_disable_fused_quant_kernels": false, - "inference_moe_token_dispatcher_type": "nvls", - "inference_rng_tracker": false, - "inference_shards": null, - "inference_text_gen_server_logging": false, - "inference_use_synchronous_zmq_collectives": false, - "inference_wandb_logging": false, - "init_method_std": 0.02, - "init_method_xavier_uniform": false, - "init_model_with_meta_device": false, - "initial_loss_scale": 4294967296, - "inprocess_active_world_size": 16, - "inprocess_barrier_timeout": 120, - "inprocess_completion_timeout": 120, - "inprocess_empty_cuda_cache": false, - "inprocess_granularity": "node", - "inprocess_hard_timeout": 90, - "inprocess_heartbeat_interval": 30, - "inprocess_heartbeat_timeout": 60, - "inprocess_last_call_wait": 1, - "inprocess_max_iterations": null, - "inprocess_monitor_process_interval": 1.0, - "inprocess_monitor_thread_interval": 1.0, - "inprocess_progress_watchdog_interval": 1.0, - "inprocess_restart": false, - "inprocess_soft_timeout": 60, - "inprocess_termination_grace_time": 1, - "is_hybrid_model": false, - "iter_per_epoch": 1250, - "iteration": 24000, - "iterations_to_skip": "[]", - "keep_fp8_transpose_cache": false, - "kitchen_attention_backend": "sdpa", - "kv_channels": 128, - "kv_lora_rank": 32, - "langrl_env_config": null, - "layernorm_epsilon": 1e-06, - "layernorm_zero_centered_gamma": false, - "lazy_mpu_init": false, - "linear_attention_freq": null, - "linear_conv_kernel_dim": 4, - "linear_cp_layout": "zigzag", - "linear_cp_mode": "headwise", - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "linear_value_head_dim": 128, - "lion_beta1": 0.95, - "lion_beta2": 0.98, - "load": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/checkpoints", - "load_main_params_from_ckpt": false, - "local_rank": 0, - "log_device_memory_used": false, - "log_energy": false, - "log_interval": 1, - "log_loss_scale_to_tensorboard": true, - "log_max_attention_logit": false, - "log_memory_interval": null, - "log_memory_to_tensorboard": false, - "log_num_zeros_in_grad": false, - "log_params_norm": false, - "log_progress": false, - "log_straggler": false, - "log_throughput": false, - "log_timers_to_tensorboard": true, - "log_validation_ppl_to_tensorboard": false, - "log_world_size_to_tensorboard": false, - "logging_level": null, - "logit_dtype": null, - "logits_load_decode_threads": 4, - "logits_load_dir": null, - "logits_load_ignore_errors": false, - "logits_load_kd_loss_alpha": 1.0, - "logits_load_msc_prefetch_depth": 2, - "logits_load_prefetch_factor": 3, - "logits_save_dir": null, - "logits_save_dtype": "fp16", - "logits_save_top_k": null, - "logits_save_top_p": null, - "logits_save_top_p_min_k": 1, - "loop_adapter_attention_backend": "sdpa_auto", - "loop_adapter_checkpoint_refiner": false, - "loop_adapter_ffn_size": 5632, - "loop_final_adapter": "lookahead_refiner", - "loop_input_write": "ifm_diagonal", - "loss_scale": null, - "loss_scale_window": 1000, - "lr": 0.007470638533717786, - "lr_decay_iters": null, - "lr_decay_samples": null, - "lr_decay_style": "linear", - "lr_warmup_fraction": null, - "lr_warmup_init": 0.0, - "lr_warmup_iters": 0, - "lr_warmup_samples": 0, - "lr_wsd_decay_iters": 0, - "lr_wsd_decay_samples": null, - "lr_wsd_decay_style": "linear", - "main_grads_dtype": "torch.float32", - "main_params_dtype": "torch.float32", - "make_vocab_size_divisible_by": 64, - "mamba_head_dim": 64, - "mamba_inference_conv_states_dtype": "torch.bfloat16", - "mamba_inference_ssm_states_dtype": "torch.bfloat16", - "mamba_num_groups": 8, - "mamba_num_heads": null, - "mamba_state_dim": 128, - "mamba_training_ssm_states_dtype": null, - "manual_gc": false, - "manual_gc_eval": true, - "manual_gc_interval": 0, - "mask_factor": 1.0, - "mask_prob": 0.15, - "mask_type": "random", - "masked_softmax_fusion": true, - "max_position_embeddings": 2048, - "max_seqlen_per_dp_cp_rank": null, - "max_tokens_to_oom": 12000, - "megatron_fsdp_enable_fine_grained_param_gather": false, - "megatron_fsdp_grad_comm_dtype": null, - "megatron_fsdp_main_grads_dtype": null, - "megatron_fsdp_main_params_dtype": "torch.float32", - "megatron_fsdp_max_pool_double_buffer": false, - "megatron_fsdp_version": 1, - "megatron_lm_path": "/root/dl/runtime/Megatron-LM", - "memory_snapshot_path": "snapshot.pickle", - "merge_file": null, - "metadata_path": null, - "mhc_fused_backend": "auto", - "mhc_init_gating_factor": 0.01, - "mhc_num_residual_streams": 4, - "mhc_recompute_layer_num": null, - "mhc_sinkhorn_iterations": 20, - "micro_batch_size": 4, - "microbatch_group_size_per_vp_stage": null, - "mid_level_dataset_surplus": 0.005, - "min_loss_scale": 1.0, - "min_lr": 0.0007470638533717787, - "min_offloaded_tensor_size": 1048576, - "mla_down_proj_fusion": false, - "mlp_chunks_for_prefill": 1, - "mlp_chunks_for_training": 1, - "mmap_bin_files": true, - "mock_data": false, - "moe_apply_probs_on_input": false, - "moe_aux_loss_coeff": 0.0, - "moe_combine_bwd_dtype": "bf16", - "moe_deepep_num_sms": null, - "moe_dispatch_fwd_dtype": "bf16", - "moe_enable_deepep": false, - "moe_enable_routing_replay": false, - "moe_expert_capacity_factor": null, - "moe_expert_rank_capacity_factor": null, - "moe_ffn_hidden_size": null, - "moe_flex_dispatcher_backend": "deepep", - "moe_flex_dispatcher_num_sms": null, - "moe_grouped_gemm": false, - "moe_hybridep_num_blocks_permute": null, - "moe_hybridep_num_blocks_unpermute": null, - "moe_hybridep_num_sms": null, - "moe_hybridep_num_sms_preprocessing": 108, - "moe_hybridep_pad_uneven_dispatch_inputs": false, - "moe_input_jitter_eps": null, - "moe_latent_size": null, - "moe_layer_freq": 1, - "moe_layer_recompute": false, - "moe_mlp_glu_interleave_size": null, - "moe_ncclep_zero_copy": false, - "moe_pad_expert_input_to_capacity": false, - "moe_pad_experts_for_cuda_graph_inference": false, - "moe_paged_stash": false, - "moe_paged_stash_buffer_size_factor_cpu": 0.0, - "moe_paged_stash_buffer_size_factor_cuda": 1.1, - "moe_paged_stash_page_size": 64, - "moe_per_layer_logging": false, - "moe_permute_fusion": false, - "moe_permute_fusion_into_hybridep": false, - "moe_router_bias_update_rate": 0.001, - "moe_router_dtype": null, - "moe_router_enable_expert_bias": false, - "moe_router_force_biased": null, - "moe_router_force_load_balancing": false, - "moe_router_fusion": false, - "moe_router_group_topk": null, - "moe_router_load_balancing_type": "aux_loss", - "moe_router_num_groups": null, - "moe_router_padding_for_fp8": false, - "moe_router_padding_for_quantization": false, - "moe_router_pre_softmax": false, - "moe_router_quantile_balancing_ema": 0.0, - "moe_router_score_function": "softmax", - "moe_router_skip_muon": false, - "moe_router_topk": 2, - "moe_router_topk_scaling_factor": null, - "moe_routing_trace_capture_hidden_states": false, - "moe_routing_trace_capture_logits": false, - "moe_routing_trace_dump_weights": false, - "moe_routing_trace_max_training_iters": null, - "moe_routing_trace_path": null, - "moe_shared_expert_gate": false, - "moe_shared_expert_glu_interleave_size": null, - "moe_shared_expert_intermediate_size": null, - "moe_shared_expert_overlap": false, - "moe_single_grouped_bias": false, - "moe_single_grouped_weight": false, - "moe_token_dispatcher_type": "allgather", - "moe_token_drop_policy": "probs", - "moe_upcycling_granularity": 1, - "moe_use_grouped_tensor": false, - "moe_use_norm_before_up_proj": false, - "moe_use_upcycling": false, - "moe_z_loss_coeff": null, - "mrope_section": null, - "mscale": 1.0, - "mscale_all_dim": 0.0, - "mtp_detach_heads": false, - "mtp_hsm": false, - "mtp_hybrid_override_pattern": null, - "mtp_loss_scaling_factor": 0.1, - "mtp_num_layers": null, - "mtp_standalone": false, - "mtp_use_repeated_layer": false, - "multi_latent_attention": false, - "multiple_validation_sets": false, - "muon_coefficient_type": "quintic", - "muon_extra_scale_factor": 1.0, - "muon_fp32_matmul_prec": "medium", - "muon_momentum": 0.95, - "muon_nesterov": true, - "muon_num_ns_steps": 5, - "muon_scalar_optimizer": "adam", - "muon_scale_mode": "spectral", - "muon_split_qkv": true, - "muon_tp_mode": "duplicated", - "muon_use_syrk": false, - "mup_attn_scale_power": 1.0, - "mup_base_head_dim": null, - "mup_base_hidden_size": null, - "mup_embedding_mult": 1.0, - "mup_output_mult": 1.0, - "mup_width_mult": 1.0, - "nccl_all_reduce_for_prefill": false, - "nccl_communicator_config_path": null, - "nccl_ub": false, - "no_load_optim": null, - "no_load_rng": null, - "no_persist_layer_norm": false, - "no_rope_freq": null, - "no_save_optim": null, - "no_save_rng": null, - "no_weight_decay_cond_type": null, - "non_persistent_ckpt_type": null, - "non_persistent_global_ckpt_dir": null, - "non_persistent_local_ckpt_algo": "fully_parallel", - "non_persistent_local_ckpt_dir": null, - "non_persistent_save_interval": null, - "normalization": "RMSNorm", - "null_tokenizer_eod_id": null, - "null_tokenizer_pad_id": -1, - "num_attention_heads": 11, - "num_channels": 3, - "num_classes": 1000, - "num_dataset_builder_threads": 1, - "num_distributed_optimizer_instances": 1, - "num_experts": null, - "num_floating_point_operations_so_far": 2.184176412603187e+19, - "num_layers": 20, - "num_layers_at_end_in_bf16": 1, - "num_layers_at_start_in_bf16": 1, - "num_layers_per_virtual_pipeline_stage": null, - "num_query_groups": 11, - "num_speculative_tokens": 0, - "num_virtual_stages_per_pipeline_rank": null, - "num_workers": 0, - "nvtx_ranges": false, - "object_storage_cache_path": null, - "offload_modules": "[]", - "offset_sampling_seed_by_dp_rank": true, - "one_logger_async": false, - "one_logger_project": "megatron-lm", - "one_logger_run_name": null, - "online_data": true, - "online_data_cache_dir": "/root/dl/runtime/script/loopTF-workspace/data/cache/huggingface", - "online_dataset": "HuggingFaceFW/fineweb-edu", - "online_dataset_revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9", - "online_dataset_subset": "sample-100BT", - "online_latency_log_dir": null, - "online_pack_sequences": false, - "online_prefetch_batches": 16, - "online_raw_manifest": null, - "online_shuffle_buffer_records": 128, - "online_source_mode": "stream", - "online_source_split": "train", - "online_split_seed": 260421106, - "online_text_column": "text", - "online_tokenizer_batch_size": 256, - "online_tokenizer_revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d", - "online_validation_fraction": 0.001, - "online_validation_manifest": "/root/dl/runtime/script/loopTF-workspace/data/validation/manual_14b/manifest.json", - "onnx_safe": null, - "openai_gelu": false, - "optimizer": "muonh", - "optimizer_cpu_offload": false, - "optimizer_cuda_graph": false, - "optimizer_offload_fraction": 1.0, - "otel_enabled": false, - "otel_service_name": null, - "otel_span_groups": null, - "outer_dp_sharding_strategy": "no_shard", - "output_bert_embeddings": false, - "overlap_cpu_optimizer_d2h_h2d": false, - "overlap_dispatch_backward_with_experts_wgrad": false, - "overlap_grad_reduce": false, - "overlap_moe_expert_parallel_comm": false, - "overlap_p2p_comm": false, - "overlap_p2p_comm_warmup_flush": false, - "overlap_param_gather": false, - "overlap_param_gather_with_optimizer_step": false, - "override_ckpt_iteration": null, - "override_opt_param_scheduler": false, - "pad_vocab_size": true, - "padded_vocab_size": 32064, - "paper_base_matrix_lr": 0.014, - "paper_embedding_lr": 0.3, - "paper_norm_lr": 0.005, - "paper_other_lr": 0.005, - "paper_reference_batch_tokens": 262144, - "paper_reference_tokens": 983315200, - "paper_reference_width": 640, - "paper_transfer": "Paper2604Transfer(width=1408, total_tokens=7000031232, batch_tokens=262144, reference_width=640, reference_tokens=983315200, reference_batch_tokens=262144)", - "paper_unembedding_lr": 0.004, - "params_dtype": "torch.bfloat16", - "patch_dim": 16, - "per_dataset_sequences_path": null, - "per_split_data_args_path": null, - "perform_initialization": true, - "perform_rl_step": false, - "phase_transition_iterations": null, - "pin_cpu_grads": true, - "pin_cpu_params": true, - "pipeline_model_parallel_comm_backend": null, - "pipeline_model_parallel_layout": null, - "pipeline_model_parallel_size": 1, - "position_embedding_type": "rope", - "pretrained_checkpoint": null, - "profile": false, - "profile_ranks": "[]", - "profile_step_end": 12, - "profile_step_start": 10, - "pytorch_profiler_collect_callstack": false, - "pytorch_profiler_collect_chakra": false, - "pytorch_profiler_collect_shapes": false, - "q_lora_rank": null, - "qk_clip": false, - "qk_clip_alpha": 0.5, - "qk_clip_threshold": 100, - "qk_head_dim": 128, - "qk_l2_norm": false, - "qk_layernorm": false, - "qk_pos_emb_head_dim": 64, - "query_in_block_prob": 0.1, - "quick_geglu": false, - "rampup_batch_size": null, - "rank": 0, - "recipe_optimizer": "{'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}", - "recipe_overrides": "{}", - "recipe_requested_batch_tokens": 262144, - "recipe_schedule": "{'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}", - "recompute_granularity": null, - "recompute_method": null, - "recompute_modules": null, - "recompute_num_layers": null, - "record_memory_history": false, - "record_shapes": false, - "refit_execution_batch_bytes": null, - "refit_method": "gloo", - "relative_attention_max_distance": 128, - "relative_attention_num_buckets": 32, - "replication": false, - "replication_factor": 2, - "replication_jump": null, - "rerun_mode": "validate_results", - "reset_attention_mask": false, - "reset_position_ids": false, - "result_rejected_tracker_filename": null, - "retriever_report_topk_accuracies": "[]", - "retriever_score_scaling": false, - "reuse_grad_buf_for_mxfp8_param_ag": false, - "rl_consumption_granularity": "B", - "rl_default_temperature": 1.0, - "rl_default_top_k": -1, - "rl_default_top_p": 0, - "rl_durable_rollout_bank": false, - "rl_generation_lag": null, - "rl_importance_sampling_truncation_coef": null, - "rl_inference_expert_model_parallel_size": null, - "rl_inference_expert_tensor_model_parallel_size": null, - "rl_inference_logprobs_is_correction": false, - "rl_inference_model_unified_memory_level": 0, - "rl_inference_parsers": "[]", - "rl_inference_pipeline_model_parallel_size": null, - "rl_inference_tensor_model_parallel_size": null, - "rl_kv_cache_management_mode": "persist", - "rl_max_inflight_requests": null, - "rl_offload_inference_model_weights_when_idle": false, - "rl_offload_optimizer_during_inference": false, - "rl_partial_rollouts": false, - "rl_persist_cuda_graphs": false, - "rl_profile": false, - "rl_profile_dir": null, - "rl_prompts_per_eval": 32, - "rl_rollout_bank_dir": null, - "rl_rollout_bank_max_bytes": 0, - "rl_sequence_packing_algo": "fifo", - "rl_sequence_packing_max_sequences_per_bin": 50, - "rl_skip_bos_token": false, - "rl_submission_granularity": "B", - "rl_training_cuda_graphs": false, - "rl_use_sequence_packing": false, - "rl_verify_model_weights_swap": false, - "rope_scaling_factor": 8.0, - "rope_type": null, - "rotary_base": 10000, - "rotary_interleaved": false, - "rotary_percent": 1.0, - "rotary_scaling_factor": 1.0, - "rotary_seq_len_interpolation_factor": null, - "run_workload_inspector_server": false, - "sample_rate": 1.0, - "save": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/checkpoints", - "save_activations_interval": null, - "save_dgrads_interval": null, - "save_interval": 4000, - "save_params_interval": null, - "save_retain_interval": null, - "save_tokenizer_assets": true, - "save_tokens_per_expert_interval": null, - "save_wgrads_interval": null, - "seed": 42, - "seq_length": 2048, - "sequence_packing_scheduler": null, - "sequence_parallel": false, - "sft": false, - "sft_mock_dataset_config_json": null, - "sft_tokenizer_prompt_format": "nemotron-h-aligned", - "sgd_momentum": 0.9, - "sharp_enabled_group": null, - "short_seq_prob": 0.1, - "skip_train": false, - "skipped_train_samples": 0, - "softmax_type": "vanilla", - "spec": null, - "special_tokens": null, - "split": null, - "squared_relu": false, - "start_eval_at_iter": null, - "start_weight_decay": 0.0, - "step_batch_size_schedule": null, - "straggler_ctrlr_port": 65535, - "straggler_minmax_count": 1, - "strict_fsdp_dtensor_load": true, - "suggested_communication_unit_size": null, - "swiglu": false, - "swin_backbone_type": "tiny", - "symmetric_ar_type": null, - "te_precision_config_file": null, - "te_rng_tracker": false, - "tensor_model_parallel_size": 1, - "tensor_parallel_num_weight_shards": 1, - "tensorboard_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/tensorboard", - "tensorboard_log_interval": 1, - "tensorboard_queue_size": 1000, - "test_data_path": null, - "test_mode": false, - "tiktoken_num_special_tokens": 1000, - "tiktoken_pattern": null, - "timing_log_level": 0, - "timing_log_option": "minmax", - "titles_data_path": null, - "tokenizer_hf_no_include_special_tokens": false, - "tokenizer_hf_no_use_fast": false, - "tokenizer_model": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "tokenizer_sentencepiece_ignore_extra_whitespaces": true, - "tokenizer_sentencepiece_legacy": false, - "tokenizer_type": "HuggingFaceTokenizer", - "torch_fsdp2_reshard_after_forward": true, - "tp_comm_bootstrap_backend": "nccl", - "tp_comm_bulk_dgrad": true, - "tp_comm_bulk_wgrad": true, - "tp_comm_overlap": false, - "tp_comm_overlap_ag": true, - "tp_comm_overlap_cfg": null, - "tp_comm_overlap_rs": true, - "tp_comm_overlap_rs_dgrad": false, - "tp_comm_split_ag": true, - "tp_comm_split_rs": true, - "train_data_path": null, - "train_iters": 26703, - "train_samples": null, - "train_sync_interval": null, - "training_recipe": "paper2604_21106", - "training_recipe_metadata": "{'recipe': 'paper2604_21106', 'width': 1408, 'total_tokens': 7000031232, 'batch_tokens': 262144, 'reference_width': 640, 'reference_tokens': 983315200, 'reference_batch_tokens': 262144, 'base_lrs': {'matrix': 0.014, 'embedding': 0.3, 'unembedding': 0.004, 'norm': 0.005, 'other': 0.005}, 'peak_lrs': {'matrix': 0.007470638533717786, 'embedding': 0.2022599587389726, 'unembedding': 0.0026967994498529684, 'norm': 0.00337099931231621, 'other': 0.00337099931231621}, 'reference': 'paper vanilla 0-20-0 at width 640; configurable token anchor', 'tied_embedding_policy': 'unembedding (one AdamW update)', 'other_parameter_policy': 'norm LR (extension for non-paper scalars/biases)', 'weight_decay': 0.0, 'warmup_steps': 0, 'final_lr_fraction': 0.1, 'optimizer': {'name': 'muonh', 'matrix_lr': 0.014, 'scalar_lr': None, 'matrix_weight_decay': 0.0, 'scalar_weight_decay': 0.0, 'embedding_lr': 0.3, 'unembedding_lr': 0.004, 'norm_lr': 0.005, 'other_lr': 0.005, 'betas': (0.8, 0.95), 'eps': 1e-10, 'normuon_momentum': 0.95, 'normuon_beta2': 0.95, 'normuon_nesterov': True, 'normuon_adjust_lr': 'spectral_norm', 'max_grad_norm': None}, 'schedule': {'kind': 'linear', 'warmup_steps': 0, 'cooldown_fraction': 0.0, 'cooldown_style': 'linear', 'final_lr_fraction': 0.1, 'z_loss_coefficient': 0.0, 'adamc_weight_decay': False}, 'group_weight_decay': {'matrix': 0.0, 'embedding': 0.0, 'unembedding': 0.0, 'norm': 0.0, 'other': 0.0}, 'recipe_overrides': {}, 'requested_batch_tokens': 262144, 'sequence_length': 2048, 'batch_assumptions': {'global_batch_tokens': 262144, 'sequence_length': 2048, 'round_up': False}, 'lr_transfer': 'mup_hyperp_released_source', 'assumptions': [], 'backend_kernel_assumptions': {}, 'paper_settings_overridden': False, 'model': {'backbone_profile': 'paper2604_vanilla', 'source_commit': '122366472a31e57982a2877eeee10bb115477713', 'width_scale': 22, 'hidden_size': 1408, 'num_layers': 20, 'head_dim': 128, 'num_attention_heads': 11, 'num_kv_heads': 11, 'ffn_hidden_size': 5632, 'activation': 'squared_relu', 'attention': 'full_causal', 'attention_output_gate': False, 'qk_norm': 'functional_rmsnorm_after_rope', 'norm_epsilon': 1e-06, 'model_level_norms': ['embedding', 'recur', 'final'], 'rope_base': 10000.0, 'rope_convention': 'split_half_negative_angle_bf16_cos_sin', 'bias': False, 'dropout': 0.0, 'residual_multiplier': 1.0, 'residual_and_norm_scale_dtype': 'float32', 'projection_dtype': 'bfloat16 (production)', 'tied_embeddings': False, 'vocab_size': 32001, 'padded_vocab_size': 32064, 'logit_softcap': 15.0, 'crop_padding_before_loss': True, 'initialization': {'embedding_normal_std': 1.0, 'head_normal_std': 0.001, 'matrix_uniform_bound': 'sqrt(3/fan_in)', 'norm_scale': 1.0}, 'parameter_counts': {'non_embedding_matrix_formula': 475791360, 'non_embedding_paper_N': 475851904, 'total_trainable': 651401344, 'feedback_parameters': 0, 'loop_input_write_parameters': 1985280, 'loop_adapter_parameters': 83271936}, 'tokenizer_fidelity_note': 'Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.', 'execution_architecture': 'full_bandwidth_loop', 'objective': 'depth_loop_causal_lm', 'feedback_enabled': False, 'train_passes': 1, 'eval_passes': 1, 'feedback_pass_mixture': {1: 1.0}, 'feedback_progressive_stages': [], 'depth_num_loops': 6, 'logical_layers_per_pass': 120, 'shared_layer_weights': True, 'depth_share_kv': False, 'depth_kv_policy': 'recompute_each_loop', 'norm_placement': 'embedding once; recur/final after loop 6 before three refiners; no post-refiner norm', 'loop_input_write': 'ifm_diagonal', 'loop_input_write_source': 'raw_token_embeddings', 'loop_input_write_placement': 'before_every_depth_loop_after_initial_norm_emb', 'loop_input_write_sharing': 'all_depth_loops_and_outer_feedback_passes', 'loop_input_write_init': 'paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)', 'loop_final_adapter': 'lookahead_refiner', 'loop_adapter': {'ffn_size': 5632, 'state_input': 'final-loop hidden states after norm_recur and norm_final', 'after_depth_loop': 6, 'applications_per_pass': 1, 'terminal_norms': True, 'attention_backend': 'sdpa_auto', 'checkpoint_refiner': False, 'bank_layers': [0, 10, 19], 'bank_source': 'first depth loop, fresh on each feedback pass', 'weight_sharing': 'one adapter shared across feedback passes'}, 'block_application_multiplier_per_pass': 6.15, 'block_application_multiplier': 6.15}}", - "transformer_impl": "local", - "transformer_pipeline_model_parallel_size": 1, - "trust_remote_code": false, - "untie_embeddings_and_output_weights": true, - "use_checkpoint_args": false, - "use_checkpoint_opt_param_scheduler": false, - "use_cpu_initialization": null, - "use_dist_ckpt": false, - "use_dist_ckpt_deprecated": false, - "use_distributed_optimizer": false, - "use_flash_attn": false, - "use_fused_mhc": false, - "use_fused_weighted_squared_relu": false, - "use_gigatoken": false, - "use_gloo_process_groups": true, - "use_grouped_gemm_for_dense_mlp": false, - "use_grouped_gemm_for_shared_expert": false, - "use_kitchen_attention": false, - "use_layer_wise_distributed_optimizer": false, - "use_layer_wise_param_layout": true, - "use_legacy_static_engine": false, - "use_mamba_mem_eff_path": true, - "use_megatron_fsdp": false, - "use_mp_args_from_checkpoint_args": false, - "use_mup": false, - "use_one_sent_docs": false, - "use_persistent_ckpt_worker": false, - "use_precision_aware_optimizer": false, - "use_pytorch_profiler": false, - "use_ring_exchange_p2p": false, - "use_rope_scaling": false, - "use_rotary_position_embeddings": false, - "use_sharp": false, - "use_te_activation_func": false, - "use_tokenizer_model_from_checkpoint_args": true, - "use_torch_fsdp2": false, - "use_torch_optimizer_for_cpu_offload": false, - "use_tp_pp_dp_mapping": false, - "use_transformer_engine_op_fuser": false, - "use_varlen_dataset": false, - "v_head_dim": 128, - "valid_data_path": null, - "validation_set_names": null, - "variable_seq_lengths": false, - "varlen_mock_dataset_config_json": null, - "varlen_sbhd_validation": false, - "verify_integrity": false, - "virtual_pipeline_model_parallel_size": null, - "vision_backbone_type": "vit", - "vision_pretraining": false, - "vision_pretraining_type": "classify", - "vocab_extra_ids": 0, - "vocab_file": null, - "vocab_size": 32001, - "wandb_entity": "LoopTF-4-CSPs", - "wandb_exp_name": "paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM", - "wandb_project": "loopTF-workspace", - "wandb_save_dir": "/root/dl/runtime/script/loopTF-workspace/output/full_bandwidth_loop/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/wandb", - "weight_decay": 0.0, - "weight_decay_incr_style": "constant", - "wgrad_deferral_limit": 0, - "window_attn_skip_freq": null, - "window_size": null, - "world_size": 16, - "yaml_cfg": null, - "yarn_beta_fast": null, - "yarn_beta_slow": null, - "yarn_correction_range_round_to_int": null, - "yarn_original_max_position_embeddings": null - }, - "native_reconstruction": { - "active_vocab_size": 32001, - "architecture": "full_bandwidth_loop", - "context_extension": "native RoPE, no scaling; at most 2x training length", - "max_supported_length": 4096, - "model_config": { - "attention_dropout": 0.0, - "attn_implementation": "sdpa", - "backbone_profile": "paper2604_vanilla", - "gradient_checkpointing": false, - "hf_config_overrides": {}, - "hidden_size": 1408, - "initializer_range": 0.02, - "intermediate_size": 5632, - "max_position_embeddings": 2048, - "model_type": "full_bandwidth", - "num_attention_heads": 11, - "num_hidden_layers": 20, - "num_key_value_heads": 11, - "padded_vocab_size": 32064, - "paper_full_attention_interval": 6, - "paper_sliding_window": 0, - "pretrained_model_name_or_path": null, - "revision": null, - "rms_norm_eps": 1e-06, - "rope_theta": 10000, - "tie_word_embeddings": false, - "trust_remote_code": false, - "vocab_size": 32001 - }, - "policy": { - "depth_loops": 6, - "depth_share_kv": false, - "feedback_layers": [ - 20 - ], - "final_adapter": { - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "ffn_size": 5632, - "kind": "lookahead_refiner" - }, - "loop_input_write": "ifm_diagonal", - "num_passes": 1 - }, - "prediction": "fixed final feedback-pass logits; eval jitter/prefix mixin disabled", - "step": 26703, - "training_context_length": 2048 - }, - "parameter_counts": { - "feedback_parameters": 0, - "loop_adapter_parameters": 83271936, - "loop_input_write_parameters": 1985280, - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 651401344 - }, - "pipeline_model_parallel_size": 1, - "run_name": "paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM", - "tensor_model_parallel_size": 1, - "tokenizer": { - "added_tokens": [ - "<|assistant_end|>" - ], - "repository_or_local_snapshot": "/taijifs_wzbi/rentTest/hunyuan/kenanchen/loopTF-workspace/data/tokenizers/paper2604_llama2_32001_3cfa8358de44148a", - "revision": "8efe6c9b93655b934e27bd9981e3ec13e55aee9d" - }, - "training_budget_tokens": 7000031232, - "training_code_revision": "82c473685b887bdf49b391763048ac06e0e3f6a3", - "training_tokens": 7000031232 -} diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env deleted file mode 100644 index bafcc85e50f18d1b502e005925e6b767bc93f01a..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/launch_config.env +++ /dev/null @@ -1,117 +0,0 @@ -# Vanilla shared-stack depth loop, L=6, with FBW completely disabled (K=1). -# Matched to 476M_7b_FBW0_K3_L2.env; see NO_FBW_BASELINES.md. -# Backbone: 475,851,904 non-embedding; with IFM and adapter: 651,401,344 total trainable parameters. -# Inject projected raw token embeddings before every loop; compute fresh KV each loop. -TRAIN_ARCHITECTURE=full_bandwidth_loop -DEPTH_NUM_LOOPS=6 -DEPTH_SHARE_KV=false -LOOP_INPUT_WRITE=ifm_diagonal -LOOP_FINAL_ADAPTER=lookahead_refiner -LOOP_ADAPTER_FFN_SIZE=5632 -LOOP_ADAPTER_ATTENTION_BACKEND=sdpa_auto -LOOP_ADAPTER_CHECKPOINT_REFINER=true -AUX_MTP_HORIZON=1 -BACKBONE_PROFILE=paper2604_vanilla -TRAINING_RECIPE=paper2604_21106 -RECIPE_OVERRIDES='{}' -# Use the installed Transformers release; the cluster's dev checkout is incompatible. -TRANSFORMERS_PATH='' -RUN_NAME=paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM -RESUME_POLICY=fresh -REQUIRE_CHECKPOINT=false - -# Online FineWeb-Edu; no prepared .bin/.idx dataset is needed. -# The launcher saves the pinned Llama-2 + assistant_end tokenizer under DATA_ROOT. -DATASET_NAME=paper2604_fineweb_edu_online_llama2_32001_t2049 -ONLINE_DATA=true -# Match the completed vanilla run: use the complete local raw cache when available. -ONLINE_SOURCE_MODE=auto -ONLINE_DATASET=HuggingFaceFW/fineweb-edu -ONLINE_DATASET_SUBSET=sample-100BT -ONLINE_DATASET_REVISION=87f09149ef4734204d70ed1d046ddc9ca3f2b8f9 -ONLINE_SOURCE_SPLIT=train -ONLINE_TEXT_COLUMN=text -ONLINE_VALIDATION_FRACTION=0.001 -ONLINE_SPLIT_SEED=260421106 -TOKENIZER_MODEL=NousResearch/Llama-2-7b-hf -ONLINE_TOKENIZER_REVISION=8efe6c9b93655b934e27bd9981e3ec13e55aee9d -ONLINE_TOKENIZER_BATCH_SIZE=256 -ONLINE_SHUFFLE_BUFFER_RECORDS=128 -NUM_WORKERS=0 -DATA_PREFLIGHT=false -EXPECTED_RECORD_LENGTH=2049 -EXPECTED_INPUT_LENGTH=2048 -MIN_MODEL_INPUT_TOKENS=7000000000 - -# Two-node control: preserve vanilla's micro/global batch using accumulation. -# The multi-node launcher supplies the node rank; preflight can override NNODES. -NNODES=2 -GPUS_PER_NODE=8 -TP=1 -PP=1 -# Two accumulation rounds on 16 GPUs: 4 sequences x 16 GPUs x 2 = 128. -MICRO_BATCH_SIZE=4 -GLOBAL_BATCH_SIZE=128 -SEQ_LENGTH=2048 -MAX_POSITION_EMBEDDINGS=2048 -# Fresh training: 26,703 updates / 7,000,031,232 physical input tokens. -# Use the vanilla LR curve from step 0 through the same 7B endpoint. -TRAIN_ITERS=26703 - -NUM_LAYERS=20 -HIDDEN_SIZE=1408 -FFN_HIDDEN_SIZE=5632 -NUM_ATTENTION_HEADS=11 -NUM_QUERY_GROUPS=11 -GROUP_QUERY_ATTENTION=false -ATTENTION_OUTPUT_GATE=false -QK_LAYERNORM=false # Profile supplies parameter-free QK RMSNorm AFTER RoPE. -SWIGLU=false # Profile supplies squared-ReLU. -DISABLE_BIAS_LINEAR=true -NORMALIZATION=RMSNorm -LAYERNORM_EPSILON=0.000001 -HIDDEN_DROPOUT=0 -ATTENTION_DROPOUT=0 -WINDOW_SIZE=none -WINDOW_ATTN_SKIP_FREQ=1 -POSITION_EMBEDDING_TYPE=rope -ROTARY_BASE=10000 -UNTIE_EMBEDDINGS=true -TOKENIZER_TYPE=HuggingFaceTokenizer -VOCAB_SIZE=32001 -MAKE_VOCAB_SIZE_DIVISIBLE_BY=64 -PRECISION=bf16 -ENABLE_RECOMPUTE=false # Required by the current loop launcher/profile guards. -MEGATRON_EXTRA_ARGS='' - -# One outer pass: ordinary next-token CE after the final depth loop. -# No feedback gate, shifted states, intermediate-loop loss, or gradient detach. -# 20 physical layers x 6 loops = 120 backbone + 3 final refiner applications. -# Normalize after the final depth loop, then run the adapter once before the head. -FEEDBACK_PASS_MIXTURE=1:1.0 -FEEDBACK_PROGRESSIVE_STAGES='[]' -FEEDBACK_EVAL_PASSES=1 -FEEDBACK_LOSS_WEIGHT=0 -FEEDBACK_JITTER_SIGMA=0 -FEEDBACK_PREFIX_MIXIN=false -FEEDBACK_Z_LOSS_COEFFICIENT=0 -FEEDBACK_LOG_HIDDEN_CHANGE=false - -# Frequent logs and a short sampled evaluation for the initial training check. -WANDB_ENTITY=LoopTF-4-CSPs -WANDB_PROJECT=loopTF-workspace -WANDB_MODE=online -WANDB_RESUME=never -WANDB_RUN_GROUP=loop_476m_7b_no_fbw_ADAPTER_IFM -WANDB_TAGS=product,loop,loop6,paper2604,476m,7b,no_fbw,K1,L6,ifm,input_injection,fresh_kv,lookahead_refiner -SAVE_INTERVAL=4000 -EVAL_AT_START=true -EVAL_INTERVAL=100 -EVAL_ITERS=2 -LOG_INTERVAL=1 -CKPT_FORMAT=torch -SEED=42 - -# Pin unpacked data and the full horizon against inherited launch settings. -ONLINE_PACK_SEQUENCES=false -EXIT_INTERVAL= diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json deleted file mode 100644 index ca6837aa28fa9f418177532a8e85005d916ff91d..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "files": { - "README.md": { - "sha256": "ea6e017150ab3f8fb8347d5d430d42897b1c966d5ca3a3169d098a9ee5ca4f98", - "size": 517 - }, - "config.json": { - "sha256": "f2e5d5ca207f4a7e2eedc1e598dacb7c0591d74e85a3708c4f8b8b4d91b5745d", - "size": 52337 - }, - "launch_config.env": { - "sha256": "7ac3f9618f4d7573586e8d58b2c50f33dea43b53ba156272e0e171764d2cebad", - "size": 4087 - }, - "model.safetensors": { - "sha256": "6322b7bdbcded79c8c91b4a9fa434fa03a03031d235fbc9cb2779e6584aa6187", - "size": 1302958576 - }, - "training_recipe.json": { - "sha256": "b68a96600f0f47099ef0abbe05fc83af3f7315dc1631c2499b5100f7ce06b965", - "size": 5132 - } - }, - "note": "manifest.json does not hash itself" -} diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors deleted file mode 100644 index 208ac2037679e5c9bd76648b33463b5679868313..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/model.safetensors +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6322b7bdbcded79c8c91b4a9fa434fa03a03031d235fbc9cb2779e6584aa6187 -size 1302958576 diff --git a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json b/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json deleted file mode 100644 index 8ba6117720916209e3a2db2f84e926f0282b323b..0000000000000000000000000000000000000000 --- a/paper2604_loop6_476m_7b_online_no_fbw_ADAPTER_IFM/step_26703/training_recipe.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "recipe": "paper2604_21106", - "width": 1408, - "total_tokens": 7000031232, - "batch_tokens": 262144, - "reference_width": 640, - "reference_tokens": 983315200, - "reference_batch_tokens": 262144, - "base_lrs": { - "matrix": 0.014, - "embedding": 0.3, - "unembedding": 0.004, - "norm": 0.005, - "other": 0.005 - }, - "peak_lrs": { - "matrix": 0.007470638533717786, - "embedding": 0.2022599587389726, - "unembedding": 0.0026967994498529684, - "norm": 0.00337099931231621, - "other": 0.00337099931231621 - }, - "reference": "paper vanilla 0-20-0 at width 640; configurable token anchor", - "tied_embedding_policy": "unembedding (one AdamW update)", - "other_parameter_policy": "norm LR (extension for non-paper scalars/biases)", - "weight_decay": 0.0, - "warmup_steps": 0, - "final_lr_fraction": 0.1, - "optimizer": { - "name": "muonh", - "matrix_lr": 0.014, - "scalar_lr": null, - "matrix_weight_decay": 0.0, - "scalar_weight_decay": 0.0, - "embedding_lr": 0.3, - "unembedding_lr": 0.004, - "norm_lr": 0.005, - "other_lr": 0.005, - "betas": [ - 0.8, - 0.95 - ], - "eps": 1e-10, - "normuon_momentum": 0.95, - "normuon_beta2": 0.95, - "normuon_nesterov": true, - "normuon_adjust_lr": "spectral_norm", - "max_grad_norm": null - }, - "schedule": { - "kind": "linear", - "warmup_steps": 0, - "cooldown_fraction": 0.0, - "cooldown_style": "linear", - "final_lr_fraction": 0.1, - "z_loss_coefficient": 0.0, - "adamc_weight_decay": false - }, - "group_weight_decay": { - "matrix": 0.0, - "embedding": 0.0, - "unembedding": 0.0, - "norm": 0.0, - "other": 0.0 - }, - "recipe_overrides": {}, - "requested_batch_tokens": 262144, - "sequence_length": 2048, - "batch_assumptions": { - "global_batch_tokens": 262144, - "sequence_length": 2048, - "round_up": false - }, - "lr_transfer": "mup_hyperp_released_source", - "assumptions": [], - "backend_kernel_assumptions": {}, - "paper_settings_overridden": false, - "model": { - "backbone_profile": "paper2604_vanilla", - "source_commit": "122366472a31e57982a2877eeee10bb115477713", - "width_scale": 22, - "hidden_size": 1408, - "num_layers": 20, - "head_dim": 128, - "num_attention_heads": 11, - "num_kv_heads": 11, - "ffn_hidden_size": 5632, - "activation": "squared_relu", - "attention": "full_causal", - "attention_output_gate": false, - "qk_norm": "functional_rmsnorm_after_rope", - "norm_epsilon": 1e-06, - "model_level_norms": [ - "embedding", - "recur", - "final" - ], - "rope_base": 10000.0, - "rope_convention": "split_half_negative_angle_bf16_cos_sin", - "bias": false, - "dropout": 0.0, - "residual_multiplier": 1.0, - "residual_and_norm_scale_dtype": "float32", - "projection_dtype": "bfloat16 (production)", - "tied_embeddings": false, - "vocab_size": 32001, - "padded_vocab_size": 32064, - "logit_softcap": 15.0, - "crop_padding_before_loss": true, - "initialization": { - "embedding_normal_std": 1.0, - "head_normal_std": 0.001, - "matrix_uniform_bound": "sqrt(3/fan_in)", - "norm_scale": 1.0 - }, - "parameter_counts": { - "non_embedding_matrix_formula": 475791360, - "non_embedding_paper_N": 475851904, - "total_trainable": 651401344, - "feedback_parameters": 0, - "loop_input_write_parameters": 1985280, - "loop_adapter_parameters": 83271936 - }, - "tokenizer_fidelity_note": "Released preprint/current code uses 32001 active tokens, padded to 32064; paper table reports 32008. Follow executable release; do not invent seven tokens.", - "execution_architecture": "full_bandwidth_loop", - "objective": "depth_loop_causal_lm", - "feedback_enabled": false, - "train_passes": 1, - "eval_passes": 1, - "feedback_pass_mixture": { - "1": 1.0 - }, - "feedback_progressive_stages": [], - "depth_num_loops": 6, - "logical_layers_per_pass": 120, - "shared_layer_weights": true, - "depth_share_kv": false, - "depth_kv_policy": "recompute_each_loop", - "norm_placement": "embedding once; recur/final after loop 6 before three refiners; no post-refiner norm", - "loop_input_write": "ifm_diagonal", - "loop_input_write_source": "raw_token_embeddings", - "loop_input_write_placement": "before_every_depth_loop_after_initial_norm_emb", - "loop_input_write_sharing": "all_depth_loops_and_outer_feedback_passes", - "loop_input_write_init": "paper_matrix; a=b_delta=0; alpha=0.5; delta=log(2)", - "loop_final_adapter": "lookahead_refiner", - "loop_adapter": { - "ffn_size": 5632, - "state_input": "final-loop hidden states after norm_recur and norm_final", - "after_depth_loop": 6, - "applications_per_pass": 1, - "terminal_norms": true, - "attention_backend": "sdpa_auto", - "checkpoint_refiner": false, - "bank_layers": [ - 0, - 10, - 19 - ], - "bank_source": "first depth loop, fresh on each feedback pass", - "weight_sharing": "one adapter shared across feedback passes" - }, - "block_application_multiplier_per_pass": 6.15, - "block_application_multiplier": 6.15 - } -}