config.json

{
  "architectures": [
    "OxMiniForCausalLM"
  ],
  "bias": false,
  "block_size": 256,
  "dropout": 0.1,
  "ffn_multiplier": 4,
  "hc_streams": 4,
  "kda_to_mla_ratio": 3,
  "mla_latent_dim": 32,
  "model_type": "oxmini",
  "n_embd": 128,
  "n_head": 4,
  "n_layer": 8,
  "rms_norm_eps": 1e-05,
  "seed": 1337,
  "tie_embeddings": true,
  "use_mhc": true,
  "use_sinkhorn_mhc": false,
  "vocab_size": 66
}