| { | |
| "variant": "fully_elastic", | |
| "vocab_size": 50432, | |
| "num_hidden_layers": 32, | |
| "hidden_size": 1344, | |
| "head_dim": 64, | |
| "base_width": 64, | |
| "mlp_base_width": 176, | |
| "widths": [ | |
| 13, | |
| 15, | |
| 17, | |
| 19, | |
| 21 | |
| ], | |
| "tiers": [ | |
| "low", | |
| "medium", | |
| "high", | |
| "extra", | |
| "max" | |
| ], | |
| "mlp_output_divisor": 2.75, | |
| "split_matrices": [ | |
| "attn.q_proj", | |
| "attn.k_proj", | |
| "attn.v_proj", | |
| "attn.gate_proj", | |
| "attn.o_proj", | |
| "mlp.gate_proj", | |
| "mlp.up_proj", | |
| "mlp.down_proj", | |
| "lm_head" | |
| ], | |
| "rope_theta": 10000.0, | |
| "rope_precision": "bfloat16", | |
| "rms_norm_eps": 1e-05, | |
| "max_position_embeddings": 2048, | |
| "tie_word_embeddings": false, | |
| "tokenizer": "EleutherAI/gpt-neox-20b", | |
| "architectures": [ | |
| "ElasticForCausalLM" | |
| ], | |
| "model_type": "elastic", | |
| "torch_dtype": "float32", | |
| "method": "Fully elastic", | |
| "scale": "375M–906M" | |
| } | |