{
  "model_type": "tiny_edu",
  "architectures": [
    "TinyEdu"
  ],
  "note": "Custom architecture, not loadable with transformers.AutoModel. Use modeling_tiny_edu.py in this repository.",
  "vocab_size": 50257,
  "tokenizer": "gpt2 (GPT-2 byte-level BPE, tiktoken / transformers 'gpt2')",
  "hidden_size": 768,
  "num_hidden_layers": 12,
  "num_attention_heads": 12,
  "num_key_value_heads": 4,
  "head_dim": 64,
  "intermediate_size": 3072,
  "hidden_act": "gelu",
  "norm": "LayerNorm (with bias), pre-norm, eps 1e-5",
  "linear_bias": false,
  "position_embedding": "rotary, theta 10000, half-split rotation",
  "tie_word_embeddings": true,
  "parameters": 114133248,
  "torch_dtype": "bfloat16",
  "context_note": "Trained context length is not recorded; in a loss-by-position test on FineWeb-Edu the loss stays flat out to 2,048 tokens."
}