| { | |
| "name": "addition_big position coupling seed 0", | |
| "task": { | |
| "max_digits": 30, | |
| "reverse": true, | |
| "zero_pad": true, | |
| "positions": "coupled", | |
| "blanks": 0, | |
| "blanks_fixed": true, | |
| "offset_max": 200, | |
| "test_digits": [ | |
| 10, | |
| 20, | |
| 30, | |
| 40, | |
| 50, | |
| 60, | |
| 80, | |
| 100, | |
| 150, | |
| 200 | |
| ], | |
| "test_examples": 256, | |
| "kind": "arithmetic" | |
| }, | |
| "model": { | |
| "d_model": 384, | |
| "n_layers": 6, | |
| "n_heads": 6, | |
| "d_mlp": 1536, | |
| "max_positions": 1024, | |
| "pos_embed": "absolute", | |
| "dropout": 0.0 | |
| }, | |
| "train": { | |
| "steps": 60000, | |
| "batch_size": 384, | |
| "lr": 0.0002, | |
| "weight_decay": 0.1, | |
| "betas": [ | |
| 0.9, | |
| 0.98 | |
| ], | |
| "warmup_steps": 2000, | |
| "cosine": true, | |
| "amp": true, | |
| "eval_every": 5000, | |
| "checkpoint_every": 20000, | |
| "seed": 0, | |
| "wandb": false | |
| } | |
| } |