File size: 942 Bytes
f6f6272 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | {
"method": "hc",
"width": 1536,
"heads": 12,
"vocab_size": 32768,
"sequence_len": 2048,
"mlp_width": 6144,
"layers": 23,
"pre_layers": 3,
"core_layers": 9,
"post_layers": 3,
"loops": 2,
"window_pattern": "L",
"max_loops": 3,
"act_epsilon": 0.01,
"ponder_weight": 0.01,
"huginn_mean": 3.0,
"huginn_sigma": 0.5,
"huginn_backprop_loops": 2,
"routing_seed": 1729,
"eval_loops": 3,
"ouro_entropy_weight": 0.1,
"ouro_gate_steps": 1024,
"ouro_exit_quantile": 0.5,
"sparse_topk": 256,
"block_size": 128,
"compression_size": 4,
"local_window": 512,
"index_heads": 4,
"index_dim": 128,
"index_rope_dim": 64,
"index_loss_weight": 1.0,
"burnin_steps": 512,
"slots": 64,
"gate_rank": 16,
"conv_size": 4,
"reference_chunk": 32,
"backend": "reference",
"scale": "formal",
"residual_streams": 4,
"residual_gate_init": 0.01,
"sinkhorn_iterations": 20,
"attnres_groups": 8
}
|