config.json

{
  "vocab": 50257,
  "sequence": 256,
  "n_layers": 4,
  "n_embed": 12288,
  "aft_kind": "conv",
  "aft_heads": 4,
  "aft_ksize": 4,
  "dyt_alpha": 0.5,
  "linear_fan_in": 4,
  "attn_fan_in": null,
  "mlp_fan_in": null,
  "tanh_linear": true,
  "tanh_mlp": true,
  "remat": true
}