| { | |
| "model_type": "tiny_edu", | |
| "architectures": [ | |
| "TinyEdu" | |
| ], | |
| "note": "Custom architecture, not loadable with transformers.AutoModel. Use modeling_tiny_edu.py in this repository.", | |
| "vocab_size": 50257, | |
| "tokenizer": "gpt2 (GPT-2 byte-level BPE, tiktoken / transformers 'gpt2')", | |
| "hidden_size": 768, | |
| "num_hidden_layers": 12, | |
| "num_attention_heads": 12, | |
| "num_key_value_heads": 4, | |
| "head_dim": 64, | |
| "intermediate_size": 3072, | |
| "hidden_act": "gelu", | |
| "norm": "LayerNorm (with bias), pre-norm, eps 1e-5", | |
| "linear_bias": false, | |
| "position_embedding": "rotary, theta 10000, half-split rotation", | |
| "tie_word_embeddings": true, | |
| "parameters": 114133248, | |
| "torch_dtype": "bfloat16", | |
| "context_note": "Trained context length is not recorded; in a loss-by-position test on FineWeb-Edu the loss stays flat out to 2,048 tokens." | |
| } |