| { |
| "model": { |
| "apply_attn_gate": false, |
| "apply_bias_term": true, |
| "apply_rmsnorm": true, |
| "arch": "huginn", |
| "attention_dropout": 0.0, |
| "attn_gate_func": "silu", |
| "causal_attn_backend": "flash", |
| "chunk_size": 1024, |
| "dense_prelude_diagonal_a_init": 1.0, |
| "dense_prelude_diagonal_dt_init": 1.0, |
| "dense_prelude_input_injection": "none", |
| "dense_prelude_input_layers": "", |
| "dense_prelude_source_layer": -1, |
| "dropout": 0.0, |
| "expert_inter_dim": 0, |
| "ffn_hidden_dim": 4352, |
| "fused_block": true, |
| "fused_output_layer": true, |
| "head_dim": 64, |
| "hidden_dropout": 0.0, |
| "huginn_coda_layers": 8, |
| "huginn_diagonal_a_init": 1.0, |
| "huginn_diagonal_dt_init": 1.0, |
| "huginn_hierarchical_h_cycles": 2, |
| "huginn_hierarchical_l_cycles": 3, |
| "huginn_hierarchical_state": "split_hl", |
| "huginn_input_injection": "diagonal", |
| "huginn_prelude_layers": 8, |
| "huginn_recurrent_layers": 12, |
| "huginn_state_init": "normal", |
| "init_embed_std": null, |
| "init_logits_std": null, |
| "init_mode": "gaussian", |
| "init_std": 0.02, |
| "layernorm_num_groups": 1, |
| "layerwise_ckpt": false, |
| "loop_diagonal_a_init": 1.0, |
| "loop_diagonal_dt_init": 1.0, |
| "loop_end_layers": 28, |
| "loop_input_injection": "none", |
| "loop_start_layer": 0, |
| "loop_times": 8, |
| "memory_efficient_norm": false, |
| "model_dim": 1536, |
| "moe_expert_backend": "sequential", |
| "moe_router_bias": false, |
| "moe_router_bias_update_rate": null, |
| "moe_router_scaling_factor": null, |
| "moe_router_score_func": "sigmoid", |
| "norm_affine": true, |
| "num_activated_experts": 0, |
| "num_activated_values": 0, |
| "num_dense_layers": null, |
| "num_experts": 0, |
| "num_heads": 24, |
| "num_kv_heads": 6, |
| "num_layers": 28, |
| "num_shared_experts": 0, |
| "num_values": 0, |
| "output_size": -1, |
| "qknorm": false, |
| "recompute_attention": true, |
| "recompute_fc1_out": true, |
| "recompute_fc3_out": true, |
| "recompute_logits": false, |
| "recompute_q": true, |
| "recompute_router": false, |
| "recompute_v": true, |
| "rmsnorm_eps": 1e-06, |
| "rope_base": 1000000.0, |
| "rope_head_dim": 64, |
| "scale_emb": false, |
| "swiglu": true, |
| "vocab_size": 250624 |
| }, |
| "tokenizer": { |
| "num_reserved_special_tokens": 0, |
| "path": "tokenizer", |
| "template_dict_json": null, |
| "type": "huggingface" |
| } |
| } |
|
|
