| { | |
| "model": { | |
| "d_model": 512, | |
| "n_layers": 6, | |
| "n_heads": 8, | |
| "n_ctx": 256, | |
| "d_ff": 1536, | |
| "dropout": 0.0, | |
| "use_moe": false | |
| }, | |
| "seed": 0, | |
| "data": { | |
| "name": "belumind/en-vi-ja-curated-500k-triplets", | |
| "splits": { | |
| "train": "train", | |
| "val": "validation", | |
| "test": "test" | |
| }, | |
| "columns": { | |
| "en": "en", | |
| "vi": "vi", | |
| "ja": "ja" | |
| }, | |
| "seed": 0, | |
| "limit": null | |
| }, | |
| "tokenizer": { | |
| "file": "unigram-16k.json", | |
| "vocab_size": 16000, | |
| "sample_triplets": 200000 | |
| }, | |
| "train": { | |
| "max_tokens": 60000000, | |
| "batch_size": 128, | |
| "lr": 0.001, | |
| "betas": [ | |
| 0.9, | |
| 0.98 | |
| ], | |
| "weight_decay": 0.1, | |
| "grad_clip": 1.0, | |
| "warmup_fraction": 0.05, | |
| "min_lr_ratio": 0.1, | |
| "eval_every_fraction": 0.1, | |
| "log_every": 50 | |
| }, | |
| "evaluate": { | |
| "batch_size": 128, | |
| "val_pairs": 4000, | |
| "max_new_tokens": 128, | |
| "bleu_limit": null | |
| }, | |
| "wandb": { | |
| "enabled": true, | |
| "project": "anlp-assignment-2" | |
| }, | |
| "name": "dense" | |
| } |