| { | |
| "vocab": 12288, | |
| "n_layer": 19, | |
| "n_embd": 768, | |
| "n_head": 12, | |
| "block": 1024, | |
| "ffn_mult": 2.8125, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-06, | |
| "pos": "rope", | |
| "rope_theta": 100000.0, | |
| "ffn": "swiglu", | |
| "value_residual": true, | |
| "qk_norm": true, | |
| "logit_cap": 0.0, | |
| "z_loss": 0.0 | |
| } |