hyperparameters.json

{
  "optimizer": "AdamW",
  "max_grad_norm": "1.0",
  "weight_decay": "0.1, 0.0",
  "betas": "(0.9, 0.95)",
  "eps": "1e-08",
  "lr": "0.0003",
  "param_groups": "2",
  "trainable_params": "10391808",
  "schedule": "linear warmup 2% then cosine to 10% of peak",
  "batch_size": "32",
  "grad_accum": "1",
  "tokens_per_step": "8192",
  "n_ctx": "256",
  "total_steps": "5137",
  "warmup_steps": "102",
  "trained_tokens": "42090496",
  "chinchilla_ratio": "0.3342",
  "precision": "fp16",
  "seed": "26"
}