| { | |
| "d_model": 32, | |
| "d_mlp": 64, | |
| "n_heads": 4, | |
| "d_head": 8, | |
| "n_layers": 2, | |
| "mlp_type": "relu", | |
| "d_vocab": 13, | |
| "n_ctx": 16, | |
| "bias": false, | |
| "norm": false, | |
| "tie_unembed": false, | |
| "model_seed": 2, | |
| "data_config": { | |
| "n_train": 5000000, | |
| "n_val": 10000, | |
| "n_test": 10000, | |
| "seed": 1234, | |
| "balanced_ops": true | |
| }, | |
| "vocab": { | |
| "plus": 10, | |
| "minus": 11, | |
| "equals": 12, | |
| "n_operand_digits": 4, | |
| "n_answer_digits": 5 | |
| }, | |
| "train_config": { | |
| "lr": 0.02, | |
| "betas": [ | |
| 0.9, | |
| 0.98 | |
| ], | |
| "weight_decay": 0.01, | |
| "batch_size": 1024, | |
| "warmup_steps": 200, | |
| "lr_schedule": "cosine", | |
| "lr_min_frac": 0.1, | |
| "eval_every_steps": 200, | |
| "seed": 2, | |
| "max_steps": null, | |
| "grad_clip": 1.0 | |
| }, | |
| "planned_repo_id": "itzPotato/arithmetic-relu-2layer-seed2" | |
| } | |