config.json

{
  "architectures": ["CompactLM"],
  "vocab_size": 12288,
  "d_model": 256,
  "n_layers": 4,
  "n_heads": 4,
  "head_dim": 64,
  "ff_dim": 640,
  "context_length": 512,
  "tie_word_embeddings": true,
  "norm_eps": 1e-05,
  "rope_base": 10000.0,
  "n_params": 6162688,
  "dtype": "float32",
  "note": "Custom LLaMA-style architecture (RoPE + causal SDPA + SwiGLU + RMSNorm pre-norm). Not transformers-native; load via the provided train_compactlm5m.py CompactLM class or a matching custom loader."
}