config/config_bert.yaml

defaults:
  - model: transformer
  - size: 1_3b
  - training: 1_3b
  - dataset: slimpajama_5m
  - benchmark: base
  - generation: base
  - _self_

hydra:
  output_subdir: null
  run:
    dir: .

mode: train
checkpoint_name: auto     # By default, sets the name to {model_name}_{size_name}
tokenizer_base: bert-base-uncased      # Defaults to GPT2 (50k vocab size)
# tokenizer_base: gpt2     # Defaults to GPT2 (50k vocab size)
seed: 42