| { |
| "architecture": "baselines.looped_transformer@LoopedTransformer", |
| "head": "lm_head@LMHead", |
| "model": { |
| "attn_branch_rms": "none", |
| "attn_branch_rms_loops": 0, |
| "attn_res": true, |
| "attn_res_max_slots": 0, |
| "attn_res_scope": "dual_axis", |
| "attn_type": "causal", |
| "bp_segment_len": 3, |
| "cycle_eps_first_extra": 0.0, |
| "cycle_eps_first_loops": 1, |
| "cycle_eps_shared_span": 0.0, |
| "cycle_eps_span_mode": "progress", |
| "cycle_eps_t0": 0.0, |
| "cycle_eps_t_floor": 1.0, |
| "cycle_eps_time_mode": "inv_h", |
| "cycle_lambda_depth_prior": 1.6, |
| "cycle_lambda_depth_prior_coef": 1.0, |
| "cycle_lambda_depth_prior_hi": 1.75, |
| "cycle_lambda_depth_prior_lo": 1.58, |
| "cycle_lambda_detach_mode": "rms7b", |
| "cycle_lambda_rms_match": false, |
| "cycle_residual_scale": true, |
| "cycle_scale_lambda": 0.5, |
| "cycle_scale_lambda_amp": 1.0, |
| "cycle_scale_lambda_end": 0.0, |
| "cycle_scale_lambda_learnable": false, |
| "cycle_scale_lambda_max": 2.5, |
| "cycle_scale_lambda_min": 1.5, |
| "cycle_scale_lambda_mode": "shared", |
| "cycle_scale_lambda_pack": "softplus", |
| "cycle_scale_lambda_sched": "none", |
| "cycle_scale_lambda_sig_eps": 0.05, |
| "dual_axis_apply": "attn", |
| "dual_axis_beta_h_decay": "none", |
| "dual_axis_beta_h_init": 0.5, |
| "dual_axis_beta_h_min": 0.0, |
| "dual_axis_beta_init": 0.5, |
| "dual_axis_beta_l_init": 0.5, |
| "dual_axis_beta_l_min": 0.0, |
| "dual_axis_beta_l_per_loop": true, |
| "dual_axis_beta_learnable": false, |
| "dual_axis_beta_p_init": 0.5, |
| "dual_axis_blend_lambda": 0.25, |
| "dual_axis_content": false, |
| "dual_axis_content_mode": "exp", |
| "dual_axis_content_temp": 1.0, |
| "dual_axis_fuse_half": false, |
| "dual_axis_fuse_mode": "sum", |
| "dual_axis_h_layout": "shared", |
| "dual_axis_h_update_every": 1, |
| "dual_axis_h_update_src": "l", |
| "dual_axis_h_write": "every_layer", |
| "dual_axis_h_write_stride": 2, |
| "dual_axis_heads": 1, |
| "dual_axis_l_persist": false, |
| "dual_axis_nd_mode": "ema", |
| "dual_axis_r_scale_mode": "none", |
| "dual_axis_residual_mode": "x_plus_r", |
| "dual_axis_update_order": "post", |
| "dual_axis_update_src": "o", |
| "dual_axis_window_size": 4, |
| "embed_inject_beta": 0.25, |
| "embed_inject_depth_scale": true, |
| "embed_inject_from_t1": false, |
| "embed_inject_learnable": false, |
| "embed_inject_mode": "loop_start_mix_soft", |
| "embed_inject_norm": "none", |
| "embed_inject_swap": false, |
| "embed_inject_t1_keep": -1.0, |
| "embed_inject_t1_style": "rms_x", |
| "embed_inject_time_mode": "inv_t", |
| "embed_inject_unit_loops": 0, |
| "embed_inject_when": "start", |
| "expansion": 4, |
| "ffn_branch_rms": "affine", |
| "ffn_type": "moe", |
| "hidden_size": 384, |
| "init_type": "lecun_normal", |
| "layernorm_scaling": false, |
| "layernorm_scaling_ell0_extra": 0.0, |
| "layernorm_scaling_ell_mode": "ell", |
| "layernorm_scaling_power": 0.5, |
| "loop_id_embed_max": 16, |
| "loop_id_embed_mode": "none", |
| "loop_id_embed_scale": 1.0, |
| "max_seq_len": 1023, |
| "melt_enabled": true, |
| "moe": { |
| "capacity_factor": 1.5, |
| "coarse_top_k": 1, |
| "expert_granularity": 8, |
| "expert_intermediate_size": 384, |
| "gate_type": "sigmoid", |
| "load_balance_bias_lr": 0.002, |
| "num_routed_experts": 24, |
| "num_shared_experts": 2, |
| "router_dtype": "float32", |
| "router_loop_bias": false, |
| "router_num_loops": 3, |
| "router_per_loop": true, |
| "router_z_loss_coef": 0.001, |
| "shared_expert": true, |
| "shared_expert_size_multiplier": 1, |
| "top_k": 6, |
| "top_k_includes_shared": true |
| }, |
| "n_layers": 6, |
| "norm_eps": 1e-06, |
| "norm_type": "pre", |
| "num_heads": 6, |
| "num_key_value_heads": 3, |
| "num_loops": 3, |
| "per_layer_out_norm": true, |
| "pos_emb_type": "rope", |
| "rope_theta": 10000.0, |
| "scale_placement": "outside", |
| "vocab_size": 49152 |
| }, |
| "tokenizer": { |
| "eos_token_id": 0, |
| "repository": "HuggingFaceTB/SmolLM-360M", |
| "revision": "59f7ef243ee09a72cbc14cb054393a3e3b771d41" |
| } |
| } |
|
|
