| defaults: | |
| - model: transformer | |
| - size: 1_3b | |
| - training: 1_3b | |
| - dataset: slimpajama_5m | |
| - benchmark: base | |
| - generation: base | |
| - _self_ | |
| hydra: | |
| output_subdir: null | |
| run: | |
| dir: . | |
| mode: train | |
| checkpoint_name: auto # By default, sets the name to {model_name}_{size_name} | |
| # tokenizer_base: bert-base-uncased # Defaults to GPT2 (50k vocab size) | |
| tokenizer_base: gpt2 # Defaults to GPT2 (50k vocab size) | |
| seed: 42 |