| { | |
| "model": { | |
| "vocab": 12288, | |
| "n_layer": 19, | |
| "n_embd": 768, | |
| "n_head": 12, | |
| "block": 1024, | |
| "ffn_mult": 2.8125, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-06, | |
| "pos": "rope", | |
| "rope_theta": 100000.0, | |
| "ffn": "swiglu", | |
| "value_residual": true, | |
| "qk_norm": true, | |
| "logit_cap": 0.0, | |
| "z_loss": 0.0 | |
| }, | |
| "train": { | |
| "seed": 1337, | |
| "micro_batch": 64, | |
| "global_batch": 512, | |
| "steps": 1000, | |
| "warmup": 50, | |
| "lr": 0.0001, | |
| "min_lr": 1e-05, | |
| "muon_lr": 0.0033333333333333335, | |
| "checkpoint_every": 500, | |
| "validate_every": 500, | |
| "log_every": 20, | |
| "compile": true | |
| }, | |
| "data": { | |
| "train": "/data/ablation/mixtures/control/train.bin", | |
| "validation": "/data/final/val.bin", | |
| "sha256": "b7f7f59462cac9092bd301e742008413ddfbbd81cb1b2e901bea0d9f9e275ecf" | |
| } | |
| } |