config.json

{
  "vocab": 12288,
  "n_layer": 19,
  "n_embd": 768,
  "n_head": 12,
  "block": 1024,
  "ffn_mult": 2.8125,
  "norm": "rmsnorm",
  "norm_eps": 1e-06,
  "pos": "rope",
  "rope_theta": 100000.0,
  "ffn": "swiglu",
  "value_residual": true,
  "qk_norm": true,
  "logit_cap": 0.0,
  "z_loss": 0.0
}