| { | |
| "model_name": "Vetox Researach Model V1", | |
| "model_type": "decoder_transformer", | |
| "vocab_size": 32000, | |
| "context_length": 2048, | |
| "d_model": 768, | |
| "n_layers": 6, | |
| "n_heads": 12, | |
| "head_dim": 64, | |
| "swiglu_hidden": 2144, | |
| "position_encoding": "learned_absolute", | |
| "normalization": "RMSNorm", | |
| "attention": "standard_MHA", | |
| "weight_tying": true, | |
| "parameter_dtype": "bfloat16", | |
| "optimizer": "AdamW", | |
| "learning_rate": 0.0001, | |
| "warmup_steps": 1000, | |
| "beta1": 0.9, | |
| "beta2": 0.95, | |
| "weight_decay": 0.1, | |
| "gradient_clip": 1.0, | |
| "train_batch_size": 4, | |
| "max_steps": 1562, | |
| "num_parameters": 69953280 | |
| } |