{
  "model": {
    "apply_attn_gate": false,
    "apply_bias_term": true,
    "apply_rmsnorm": true,
    "arch": "huginn",
    "attention_dropout": 0.0,
    "attn_gate_func": "silu",
    "causal_attn_backend": "flash",
    "chunk_size": 1024,
    "dense_prelude_diagonal_a_init": 1.0,
    "dense_prelude_diagonal_dt_init": 1.0,
    "dense_prelude_input_injection": "none",
    "dense_prelude_input_layers": "",
    "dense_prelude_source_layer": -1,
    "dropout": 0.0,
    "expert_inter_dim": 0,
    "ffn_hidden_dim": 4352,
    "fused_block": true,
    "fused_output_layer": true,
    "head_dim": 64,
    "hidden_dropout": 0.0,
    "huginn_coda_layers": 8,
    "huginn_diagonal_a_init": 1.0,
    "huginn_diagonal_dt_init": 1.0,
    "huginn_hierarchical_h_cycles": 2,
    "huginn_hierarchical_l_cycles": 3,
    "huginn_hierarchical_state": "split_hl",
    "huginn_input_injection": "diagonal",
    "huginn_prelude_layers": 8,
    "huginn_recurrent_layers": 12,
    "huginn_state_init": "normal",
    "init_embed_std": null,
    "init_logits_std": null,
    "init_mode": "gaussian",
    "init_std": 0.02,
    "layernorm_num_groups": 1,
    "layerwise_ckpt": false,
    "loop_diagonal_a_init": 1.0,
    "loop_diagonal_dt_init": 1.0,
    "loop_end_layers": 28,
    "loop_input_injection": "none",
    "loop_start_layer": 0,
    "loop_times": 8,
    "memory_efficient_norm": false,
    "model_dim": 1536,
    "moe_expert_backend": "sequential",
    "moe_router_bias": false,
    "moe_router_bias_update_rate": null,
    "moe_router_scaling_factor": null,
    "moe_router_score_func": "sigmoid",
    "norm_affine": true,
    "num_activated_experts": 0,
    "num_activated_values": 0,
    "num_dense_layers": null,
    "num_experts": 0,
    "num_heads": 24,
    "num_kv_heads": 6,
    "num_layers": 28,
    "num_shared_experts": 0,
    "num_values": 0,
    "output_size": -1,
    "qknorm": false,
    "recompute_attention": true,
    "recompute_fc1_out": true,
    "recompute_fc3_out": true,
    "recompute_logits": false,
    "recompute_q": true,
    "recompute_router": false,
    "recompute_v": true,
    "rmsnorm_eps": 1e-06,
    "rope_base": 1000000.0,
    "rope_head_dim": 64,
    "scale_emb": false,
    "swiglu": true,
    "vocab_size": 250624
  },
  "tokenizer": {
    "num_reserved_special_tokens": 0,
    "path": "tokenizer",
    "template_dict_json": null,
    "type": "huggingface"
  }
}