| { | |
| "architecture": "TransformerLM", | |
| "model_type": "custom-decoder-only-transformer", | |
| "tokenizer": "tiktoken-gpt2-bpe", | |
| "weight_tying": true, | |
| "pre_layernorm": true, | |
| "positional_encoding": "sinusoidal", | |
| "activation": "gelu", | |
| "vocab_size": 50257, | |
| "d_model": 256, | |
| "n_heads": 4, | |
| "n_layers": 6, | |
| "d_ff": 1024, | |
| "max_seq_len": 256, | |
| "dropout": 0.1, | |
| "training": { | |
| "dataset": "Skylion007/openwebtext", | |
| "steps": 17999, | |
| "val_loss": 4.986894807815552 | |
| } | |
| } |