| { | |
| "model_name": "Intelligence_OS", | |
| "version": "0.1-research-prototype", | |
| "training_initialization": "random", | |
| "model_type": "decoder-only sparse transformer", | |
| "vocab_size": 49152, | |
| "d_model": 2048, | |
| "n_layers": 26, | |
| "n_query_heads": 16, | |
| "n_key_value_heads": 4, | |
| "head_dim": 128, | |
| "n_experts_per_layer": 4, | |
| "top_k_experts": 2, | |
| "expert_hidden": 1792, | |
| "max_sequence_length": 4096, | |
| "position_encoding": "RoPE", | |
| "normalization": "pre-norm RMSNorm", | |
| "feed_forward": "SwiGLU MoE", | |
| "tied_embeddings": true, | |
| "theoretical_parameter_count": 1518659584, | |
| "status": "architecture scaffold; full training system not yet implemented" | |
| } | |