{
  "model_type": "talos_gpt",
  "model_name": "Styx-100M",
  "parameters": 96482304,
  "vocab_size": 1024,
  "max_seq_len": 512,
  "hidden_size": 1024,
  "num_layers": 6,
  "num_attention_heads": 64,
  "num_kv_heads": 32,
  "head_dim": 16,
  "ffn_type": "dense",
  "intermediate_size": 4096,
  "rope_theta": 10000,
  "attention": "full",
  "dropout": 0,
  "layer_norm_eps": 1e-05,
  "tie_word_embeddings": false,
  "eos_token_id": 1021,
  "pad_token_id": 1022,
  "training_tokens": 600001892,
  "training_step": 293543,
  "train_loss": 1.3477928638458252,
  "validation_loss": 1.3706818938122054,
  "tokenizer_sha256": "58e4ad40b174c9fde3cec8e86188115bcdee173e7fcd188e7ba139a04b549e48"
}