| { | |
| "model": "Talos 10M", | |
| "architecture": "TalosGPT", | |
| "parameters": 9952320, | |
| "vocab_size": 1024, | |
| "training_tokenizer_vocab_size": 512, | |
| "hidden_size": 448, | |
| "num_layers": 3, | |
| "num_attention_heads": 28, | |
| "num_kv_heads": 14, | |
| "head_dim": 16, | |
| "intermediate_size": 1792, | |
| "max_seq_len": 512, | |
| "attention_type": "full", | |
| "rope_theta": 10000, | |
| "training_steps": 100000, | |
| "validation_loss": 1.1523912596702575, | |
| "seed": 1337 | |
| } |