| { | |
| "d_model": 256, | |
| "num_heads": 8, | |
| "num_encoder_layers": 4, | |
| "num_decoder_layers": 4, | |
| "d_ff": 1024, | |
| "dropout": 0.1, | |
| "label_smoothing": 0.1, | |
| "learning_rate": 0.0003, | |
| "warmup_steps": 2000, | |
| "batch_size": 64, | |
| "grad_clip_norm": 1.0, | |
| "max_seq_len": 1184, | |
| "seed": 42, | |
| "epochs": 50, | |
| "patience": 5, | |
| "bpe_vocab_size": 1000, | |
| "positional_encoding": "sinusoidal", | |
| "attention_type": "mha", | |
| "norm_type": "layernorm", | |
| "tokenization": "blt", | |
| "gqa_kv_heads": 2, | |
| "blt_patch_size": 9, | |
| "d_local": 128, | |
| "num_local_layers": 2, | |
| "run_name": "c5", | |
| "num_params": 7771907, | |
| "device": "cuda" | |
| } |