| { | |
| "model": { | |
| "d_model": 512, | |
| "n_layers": 6, | |
| "n_heads": 8, | |
| "n_ctx": 256, | |
| "d_ff": 1536, | |
| "dropout": 0.0 | |
| }, | |
| "seed": 0, | |
| "data": { | |
| "name": "browndw/human-ai-parallel-corpus", | |
| "fractions": { | |
| "val": 0.05, | |
| "test": 0.05 | |
| }, | |
| "seed": 0, | |
| "limit": null | |
| }, | |
| "tokenizer": { | |
| "file": "unigram-16k.json", | |
| "vocab_size": 16000 | |
| }, | |
| "train": { | |
| "batch_size": 32, | |
| "grad_clip": 0.0, | |
| "warmup_fraction": 0.05, | |
| "min_lr_ratio": 0.1, | |
| "eval_every_fraction": 0.1, | |
| "log_every": 20 | |
| }, | |
| "evaluate": { | |
| "batch_size": 64, | |
| "prompt_tokens": 128, | |
| "reference_tokens": 64 | |
| }, | |
| "wandb": { | |
| "enabled": true, | |
| "project": "anlp-assignment-2" | |
| }, | |
| "name": "mars-half", | |
| "optimizer": { | |
| "name": "mars", | |
| "lr": 0.002, | |
| "betas": [ | |
| 0.9, | |
| 0.95 | |
| ], | |
| "gamma": 0.025, | |
| "eps": 1e-10, | |
| "weight_decay": 0.1, | |
| "grad_norm": 1.0 | |
| } | |
| } |