File size: 1,113 Bytes
7262f08 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | {
"architecture": "dense",
"dense_layers": 24,
"pre_layers": 3,
"core_layers": 9,
"post_layers": 3,
"recursions": 1,
"activation_checkpointing": false,
"fourier_dim": 256,
"loop_modulation_bound": 0.5,
"loop_condition_lr_multiplier": 0.5,
"loop_condition_grad_clip": 1.0,
"vocab_size": 32768,
"sequence_len": 2048,
"width": 1536,
"heads": 12,
"mlp_width": 6144,
"window_pattern": "SSSL",
"circle_max_steps": 27,
"circle_target_steps": 17.5,
"circle_method": "sparse_gate",
"circle_budget_mode": "mean_survival",
"circle_halter": "delta_depth",
"circle_halter_eps": 1e-06,
"circle_exit_interval": 1,
"circle_threshold": 0.5,
"circle_routing_seed": 1729,
"circle_budget_beta": 0.9,
"circle_random_warmup": 256,
"circle_route_transition": 768,
"circle_depth_rho": 1.0,
"circle_shape_weight": 0.1,
"circle_dual_lr": 0.05,
"circle_dual_limit": 2.0,
"circle_halter_lr_multiplier": 0.5,
"circle_halter_grad_clip": 1.0,
"mor_target_survival": [
0.64,
0.32
],
"mor_budget_strength": 1.0,
"mor_dual_lr": 0.05,
"mor_dual_limit": 10.0
}
|