modular_mlp/config.json

{
  "name": "modular_mlp",
  "task": {
    "p": 97,
    "train_frac": 0.3,
    "kind": "modular"
  },
  "model": {
    "arch": "mlp",
    "d_model": 128,
    "n_layers": 1,
    "n_heads": 4,
    "d_mlp": 256,
    "max_positions": 128,
    "pos_embed": "absolute",
    "dropout": 0.0
  },
  "train": {
    "steps": 200000,
    "batch_size": 0,
    "lr": 0.001,
    "weight_decay": 1.0,
    "betas": [
      0.9,
      0.98
    ],
    "warmup_steps": 0,
    "cosine": false,
    "amp": false,
    "eval_every": 500,
    "checkpoint_every": 500,
    "seed": 0,
    "wandb": false
  }
}