{
  "model": {
    "vocab": 12288,
    "n_layer": 19,
    "n_embd": 768,
    "n_head": 12,
    "block": 1024,
    "ffn_mult": 2.8125,
    "norm": "rmsnorm",
    "norm_eps": 1e-06,
    "pos": "rope",
    "rope_theta": 100000.0,
    "ffn": "swiglu",
    "value_residual": true,
    "qk_norm": true,
    "logit_cap": 0.0,
    "z_loss": 0.0
  },
  "train": {
    "seed": 1337,
    "micro_batch": 64,
    "global_batch": 512,
    "steps": 1000,
    "warmup": 50,
    "lr": 0.0001,
    "min_lr": 1e-05,
    "muon_lr": 0.0033333333333333335,
    "checkpoint_every": 500,
    "validate_every": 500,
    "log_every": 20,
    "compile": true
  },
  "data": {
    "train": "/data/ablation/mixtures/control/train.bin",
    "validation": "/data/final/val.bin",
    "sha256": "b7f7f59462cac9092bd301e742008413ddfbbd81cb1b2e901bea0d9f9e275ecf"
  }
}