{
  "d_model": 32,
  "d_mlp": 64,
  "n_heads": 4,
  "d_head": 8,
  "n_layers": 2,
  "mlp_type": "relu",
  "d_vocab": 13,
  "n_ctx": 16,
  "bias": false,
  "norm": false,
  "tie_unembed": false,
  "model_seed": 2,
  "data_config": {
    "n_train": 5000000,
    "n_val": 10000,
    "n_test": 10000,
    "seed": 1234,
    "balanced_ops": true
  },
  "vocab": {
    "plus": 10,
    "minus": 11,
    "equals": 12,
    "n_operand_digits": 4,
    "n_answer_digits": 5
  },
  "train_config": {
    "lr": 0.02,
    "betas": [
      0.9,
      0.98
    ],
    "weight_decay": 0.01,
    "batch_size": 1024,
    "warmup_steps": 200,
    "lr_schedule": "cosine",
    "lr_min_frac": 0.1,
    "eval_every_steps": 200,
    "seed": 2,
    "max_steps": null,
    "grad_clip": 1.0
  },
  "planned_repo_id": "itzPotato/arithmetic-relu-2layer-seed2"
}