| { | |
| "d": 512, | |
| "vocab_size": 49152, | |
| "num_heads": 8, | |
| "head_dim": 64, | |
| "mlp_hidden_dim": 1408, | |
| "diff_blocks_num": 3, | |
| "layers_per_block": 6, | |
| "rope_theta": 50000.0, | |
| "max_position_embeddings": 4096, | |
| "mask_token_id": 49151, | |
| "max_insert": 4, | |
| "rms_eps": 1e-06, | |
| "attn_backend": "sdpa" | |
| } |