config.json

{
  "block": 1024,
  "ffn": "swiglu",
  "ffn_mult": 2.667,
  "logit_cap": 0.0,
  "n_embd": 960,
  "n_head": 15,
  "n_layer": 20,
  "norm": "rmsnorm",
  "norm_eps": 1e-06,
  "pos": "rope",
  "qk_norm": true,
  "rope_theta": 100000.0,
  "tied_weights": {
    "tok.weight": "head.weight"
  },
  "value_residual": true,
  "vocab": 32768
}