| { | |
| "architectures": ["CompactLM"], | |
| "vocab_size": 12288, | |
| "d_model": 256, | |
| "n_layers": 4, | |
| "n_heads": 4, | |
| "head_dim": 64, | |
| "ff_dim": 640, | |
| "context_length": 512, | |
| "tie_word_embeddings": true, | |
| "norm_eps": 1e-05, | |
| "rope_base": 10000.0, | |
| "n_params": 6162688, | |
| "dtype": "float32", | |
| "note": "Custom LLaMA-style architecture (RoPE + causal SDPA + SwiGLU + RMSNorm pre-norm). Not transformers-native; load via the provided train_compactlm5m.py CompactLM class or a matching custom loader." | |
| } |