| { | |
| "architectures": ["CompactLM"], | |
| "vocab_size": 12288, | |
| "d_model": 256, | |
| "n_layers": 4, | |
| "n_heads": 4, | |
| "ffn_dim": 640, | |
| "max_position_embeddings": 512, | |
| "tie_word_embeddings": true, | |
| "rms_norm": true, | |
| "rope": true, | |
| "swiglu": true, | |
| "num_parameters": 6162688 | |
| } |