{
  "model_type": "dwslm",
  "architecture": "DWSLM",
  "model": {
    "vocab": 32768,
    "d": 1024,
    "groups": 5,
    "local": "kda",
    "heads": 8,
    "kv_heads": 2,
    "head_dim": 128,
    "window": 512,
    "lin_heads": 8,
    "conv": 4,
    "mlp_ratio": 4,
    "softcap": 20.0,
    "route_heads": 8,
    "init_mult": 0.5,
    "drop": [],
    "mlp_prec": "bf16",
    "attn": "flash",
    "graphs": "none",
    "kda_gate": "clamp",
    "kda_bwd": "recompute",
    "yarn": 1.0,
    "rope_len": 4096,
    "ve_every": 2
  },
  "untied": true,
  "context": 4096,
  "vocab_size": 32768,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "pad_token_id": 0,
  "torch_dtype": "float32",
  "parameters": {
    "total": 654822280,
    "compute": 285800000,
    "lookup": 369100000
  },
  "train": {
    "step": 45030,
    "variant": "base",
    "commit": "14043b4"
  }
}