model_config.json

{
  "architecture": "McConversationBotV3",
  "architecture_base": "V15-compatible",
  "bias": false,
  "block_size": 512,
  "default_checkpoint": "model.pt",
  "dropout": 0.05,
  "gqa": false,
  "mlp_hidden_size": 1728,
  "mlp_ratio": 2.6666667,
  "model_family": "McConversationBot",
  "model_name": "McConversationBot",
  "model_version": "V3",
  "n_embed": 640,
  "n_head": 8,
  "n_kv_head": 8,
  "n_layer": 12,
  "parameter_count": 62113280,
  "rmsnorm": true,
  "rope": true,
  "rope_theta": 10000.0,
  "special_tokens": {
    "<|assistant|>": 4097,
    "<|end|>": 4098,
    "<|user|>": 4096
  },
  "swiglu": true,
  "use_bias": false,
  "use_gqa": false,
  "use_rmsnorm": true,
  "use_rope": true,
  "use_sdpa": true,
  "use_swiglu": true,
  "vocab_size": 4099
}