| { |
| "architecture": "dense", |
| "dense_layers": 24, |
| "pre_layers": 3, |
| "core_layers": 9, |
| "post_layers": 3, |
| "recursions": 1, |
| "activation_checkpointing": false, |
| "fourier_dim": 256, |
| "loop_modulation_bound": 0.5, |
| "loop_condition_lr_multiplier": 0.5, |
| "loop_condition_grad_clip": 1.0, |
| "vocab_size": 32768, |
| "sequence_len": 2048, |
| "width": 1536, |
| "heads": 12, |
| "mlp_width": 6144, |
| "window_pattern": "SSSL", |
| "circle_max_steps": 27, |
| "circle_target_steps": 17.5, |
| "circle_method": "sparse_gate", |
| "circle_budget_mode": "mean_survival", |
| "circle_halter": "delta_depth", |
| "circle_halter_eps": 1e-06, |
| "circle_exit_interval": 1, |
| "circle_threshold": 0.5, |
| "circle_routing_seed": 1729, |
| "circle_budget_beta": 0.9, |
| "circle_random_warmup": 256, |
| "circle_route_transition": 768, |
| "circle_depth_rho": 1.0, |
| "circle_shape_weight": 0.1, |
| "circle_dual_lr": 0.05, |
| "circle_dual_limit": 2.0, |
| "circle_halter_lr_multiplier": 0.5, |
| "circle_halter_grad_clip": 1.0, |
| "mor_target_survival": [ |
| 0.64, |
| 0.32 |
| ], |
| "mor_budget_strength": 1.0, |
| "mor_dual_lr": 0.05, |
| "mor_dual_limit": 10.0 |
| } |
|
|
