OpenSML-Base-175M / config.json
wzebrowski's picture
Add base model card, tokenizer, native runner and pretraining records
ed99a2e verified
Raw History Blame Contribute Delete
753 Bytes
{
"model_name": "OpenSML-Base-175M",
"parameters": 175611648,
"model": {
"attention_impl": "fast",
"bias": false,
"ce_impl": "metal",
"d_model": 768,
"ffn_impl": "packed-metal",
"loss_dtype": "float32",
"max_seq_len": 2048,
"mlp_multiple_of": 256,
"mlp_ratio": 4.0,
"n_heads": 12,
"n_kv_heads": 4,
"n_layers": 24,
"qk_norm": true,
"vocab_size": 32000,
"rope_base": 10000.0
},
"weight_sha256": "5151567e266244c9e32fd6e01775192633abe227320269ecdefe28ce43e9ccc3",
"weight_format": "BF16 safetensors",
"context_limit": 2048,
"serialization": "raw document-prefix completion",
"supported_runtime": "native MLX on Apple Silicon; not a Transformers or mlx-lm auto-loading export"
}