| { | |
| "vocab_size": 32768, | |
| "hidden_size": 128, | |
| "num_layers": 4, | |
| "max_seq_len": 512, | |
| "rms_eps": 1e-06, | |
| "full_attn_every": 4, | |
| "num_heads": 2, | |
| "q_lora_rank": 64, | |
| "kv_lora_rank": 32, | |
| "qk_nope_head_dim": 32, | |
| "qk_rope_head_dim": 16, | |
| "v_head_dim": 64, | |
| "rope_theta": 10000.0, | |
| "kda_head_dim": 64, | |
| "kda_conv_size": 4, | |
| "kda_gate_lower_bound": -5.0, | |
| "num_experts": 8, | |
| "experts_per_token": 2, | |
| "num_shared_experts": 1, | |
| "moe_latent_size": 96, | |
| "moe_intermediate_size": 64, | |
| "dense_intermediate_size": 512, | |
| "first_k_dense": 1, | |
| "num_expert_groups": 2, | |
| "topk_groups": 1, | |
| "routed_scaling": 2.5, | |
| "swiglu_limit": 10.0, | |
| "moe_capacity_factor": 2.0, | |
| "hc_mult": 4, | |
| "hc_sinkhorn_iters": 20, | |
| "attn_res_block_size": 2, | |
| "engram_layers": [ | |
| 1 | |
| ], | |
| "engram_max_ngram": 3, | |
| "engram_table_size": 65521, | |
| "engram_heads": 2, | |
| "engram_head_dim": 32 | |
| } |