config/training/large.yaml

defaults:
  - _base
  - _self_

# Identity
training_name: large
use_ddp: true

# Training
max_epochs: 1
warmup_steps: 500

# Precision and Compilation
precision: bfloat16
compile: true
compile_mode: default

# Learning Rate + LR Search
lr: 2e-4

# Data Loading
batch_size: 32
num_workers: 8
use_sliding_window: true

# Gradient Accumulation and Step Tracking
use_grad_accum: true
grad_accum_steps: auto
tokens_per_step: 1048576     # ~1m (Note that this is not perfectly enforced and may be adjusted based on batch size and world size)

# Validation Steps
validation_steps: 1000