| { | |
| "eos_id": 50282, | |
| "expanded_width": 2624, | |
| "format_version": 1, | |
| "global_every": 3, | |
| "global_rope_base": 160000.0, | |
| "head_activation": "gelu", | |
| "heads": 16, | |
| "layers": 29, | |
| "local_rope_base": 160000.0, | |
| "local_window": 64, | |
| "maximum_length": 7999, | |
| "norm_epsilon": 1e-05, | |
| "pad_id": 50283, | |
| "storage_vocabulary_size": 65536, | |
| "vocabulary_size": 50368, | |
| "width": 1024 | |
| } | |