{
  "model_name": "Vetox Researach Model V1",
  "model_type": "decoder_transformer",
  "vocab_size": 32000,
  "context_length": 2048,
  "d_model": 768,
  "n_layers": 6,
  "n_heads": 12,
  "head_dim": 64,
  "swiglu_hidden": 2144,
  "position_encoding": "learned_absolute",
  "normalization": "RMSNorm",
  "attention": "standard_MHA",
  "weight_tying": true,
  "parameter_dtype": "bfloat16",
  "optimizer": "AdamW",
  "learning_rate": 0.0001,
  "warmup_steps": 1000,
  "beta1": 0.9,
  "beta2": 0.95,
  "weight_decay": 0.1,
  "gradient_clip": 1.0,
  "train_batch_size": 4,
  "max_steps": 1562,
  "num_parameters": 69953280
}