| { | |
| "d_model": 512, | |
| "enc_layers": 12, | |
| "dec_layers": 2, | |
| "n_heads": 8, | |
| "ffn_dim": 2048, | |
| "max_len": 256, | |
| "pad_id": 0, | |
| "bos_id": 2, | |
| "eos_id": 3, | |
| "vocab_size": 32000, | |
| "pos_encoding": "rope", | |
| "norm": "rmsnorm", | |
| "tie_all_embeddings": true, | |
| "lang_markers": { | |
| "en": "<2en>", | |
| "km": "<2km>" | |
| }, | |
| "dtype": "float16" | |
| } |