| { | |
| "model_type": "dwslm", | |
| "architecture": "DWSLM", | |
| "model": { | |
| "vocab": 32768, | |
| "d": 1024, | |
| "groups": 5, | |
| "local": "kda", | |
| "heads": 8, | |
| "kv_heads": 2, | |
| "head_dim": 128, | |
| "window": 512, | |
| "lin_heads": 8, | |
| "conv": 4, | |
| "mlp_ratio": 4, | |
| "softcap": 20.0, | |
| "route_heads": 8, | |
| "init_mult": 0.5, | |
| "drop": [], | |
| "mlp_prec": "bf16", | |
| "attn": "flash", | |
| "graphs": "none", | |
| "kda_gate": "clamp", | |
| "kda_bwd": "recompute", | |
| "yarn": 1.0, | |
| "rope_len": 4096, | |
| "ve_every": 2 | |
| }, | |
| "untied": true, | |
| "context": 4096, | |
| "vocab_size": 32768, | |
| "bos_token_id": 1, | |
| "eos_token_id": 2, | |
| "pad_token_id": 0, | |
| "torch_dtype": "float32", | |
| "parameters": { | |
| "total": 654822280, | |
| "compute": 285800000, | |
| "lookup": 369100000 | |
| }, | |
| "train": { | |
| "step": 45030, | |
| "variant": "base", | |
| "commit": "14043b4" | |
| } | |
| } | |