File size: 834 Bytes
79fa441 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | {
"model": {
"vocab": 12288,
"n_layer": 19,
"n_embd": 768,
"n_head": 12,
"block": 1024,
"ffn_mult": 2.8125,
"norm": "rmsnorm",
"norm_eps": 1e-06,
"pos": "rope",
"rope_theta": 100000.0,
"ffn": "swiglu",
"value_residual": true,
"qk_norm": true,
"logit_cap": 0.0,
"z_loss": 0.0
},
"train": {
"seed": 1337,
"micro_batch": 64,
"global_batch": 512,
"steps": 1000,
"warmup": 50,
"lr": 0.0001,
"min_lr": 1e-05,
"muon_lr": 0.0033333333333333335,
"checkpoint_every": 500,
"validate_every": 500,
"log_every": 20,
"compile": true
},
"data": {
"train": "/data/ablation/mixtures/control/train.bin",
"validation": "/data/final/val.bin",
"sha256": "b7f7f59462cac9092bd301e742008413ddfbbd81cb1b2e901bea0d9f9e275ecf"
}
} |