{
  "model": {
    "apply_attn_gate": false,
    "apply_bias_term": true,
    "apply_rmsnorm": true,
    "arch": "transformer",
    "attention_dropout": 0.0,
    "attn_act_func": "softmax",
    "attn_gate_func": "silu",
    "awm_orthogonal_update": false,
    "causal_attn_backend": "flash",
    "causal_conv_backend": "triton",
    "causal_conv_weight_normalization": true,
    "causal_conv_width": 4,
    "chunk_size": 1024,
    "ddp_backend": "fsdp1",
    "dense_prelude_diagonal_a_init": 1.0,
    "dense_prelude_diagonal_dt_init": 1.0,
    "dense_prelude_input_injection": "none",
    "dense_prelude_input_layers": "",
    "dense_prelude_source_layer": -1,
    "dropout": 0.0,
    "expert_inter_dim": 0,
    "ffn_hidden_dim": 8704,
    "fused_block": false,
    "fused_output_layer": true,
    "head_dim": 64,
    "hidden_dropout": 0.0,
    "huginn_antithetic_sampling": false,
    "huginn_backprop_depth": null,
    "huginn_coda_layers": 1,
    "huginn_depth_control": false,
    "huginn_depth_prior": "none",
    "huginn_depth_prior_entropy": 0.0,
    "huginn_diagonal_a_init": 1.0,
    "huginn_diagonal_dt_init": 1.0,
    "huginn_hierarchical_h_cycles": 2,
    "huginn_hierarchical_l_cycles": 3,
    "huginn_hierarchical_state": "none",
    "huginn_input_injection": "diagonal",
    "huginn_ortho_projection_eps": 1e-06,
    "huginn_poisson_lognormal_max": 64,
    "huginn_poisson_lognormal_sigma": 0.5,
    "huginn_poisson_lognormal_target_mean": 5.0,
    "huginn_prelude_layers": 1,
    "huginn_prelude_norm": "none",
    "huginn_prelude_orthogonal": false,
    "huginn_recurrent_exit_norm": "none",
    "huginn_recurrent_layers": null,
    "huginn_sampling_scheme": "fixed",
    "huginn_split_module_repeats": 1,
    "huginn_state_init": "zero",
    "init_embed_std": null,
    "init_logits_std": null,
    "init_mode": "gaussian",
    "init_std": 0.02,
    "layernorm_eps": 1e-05,
    "layernorm_num_groups": 1,
    "layerwise_ckpt": false,
    "loop_diagonal_a_init": 1.0,
    "loop_diagonal_dt_init": 1.0,
    "loop_end_layers": null,
    "loop_input_injection": "none",
    "loop_start_layer": 0,
    "loop_times": 1,
    "memory_efficient_norm": false,
    "model_dim": 3072,
    "moe_expert_backend": "sequential",
    "moe_permutation_backend": "torch",
    "moe_router_bias": false,
    "moe_router_bias_update_rate": null,
    "moe_router_scaling_factor": null,
    "moe_router_score_func": "sigmoid",
    "mova_backend": "sequential",
    "norm_affine": true,
    "num_activated_experts": 0,
    "num_activated_values": 0,
    "num_dense_layers": null,
    "num_experts": 0,
    "num_heads": 48,
    "num_kv_heads": 12,
    "num_layers": 4,
    "num_shared_experts": 0,
    "num_values": 0,
    "output_size": -1,
    "qknorm": false,
    "recompute_attention": false,
    "recompute_awk": false,
    "recompute_fc1_out": false,
    "recompute_fc3_out": false,
    "recompute_logits": false,
    "recompute_q": false,
    "recompute_router": false,
    "recompute_v": false,
    "residual_func": "base",
    "residual_heads": null,
    "rmsnorm_eps": 1e-06,
    "rope_base": 1000000.0,
    "rope_head_dim": 64,
    "sca_backend": "swift",
    "scale_emb": false,
    "swiglu": true,
    "timenorm_backend": "cub",
    "timenorm_beta1": 0.999,
    "timenorm_beta2": 0.9999,
    "timenorm_eps": 1e-05,
    "timenorm_num_groups": 32,
    "v_head_dim": null,
    "vocab_size": 64256
  },
  "tokenizer": {
    "path": "tokenizer",
    "type": "huggingface"
  }
}