| { | |
| "angle_init_scale": 0.1, | |
| "architectures": [ | |
| "GRTForCausalLM" | |
| ], | |
| "bos_token_id": 2, | |
| "coupling_init_scale": 0.01, | |
| "d_ff": null, | |
| "d_model": 256, | |
| "dropout": 0.0, | |
| "eos_token_id": 3, | |
| "ffn_mult": 4, | |
| "ffn_n_rotations": 9, | |
| "learnable_rope": true, | |
| "log_magnitude_init": 0.0, | |
| "max_seq_len": 1024, | |
| "model_type": "grt", | |
| "n_blocks": 6, | |
| "n_heads": 4, | |
| "n_mixer_rotations": 1, | |
| "n_qkv_rotations": 5, | |
| "num_hidden_layers": 6, | |
| "output_scale_init": 30.0, | |
| "pad_token_id": 0, | |
| "phase_init_scale": 0.1, | |
| "pos_freq_base": 10000.0, | |
| "residual_gate_mode": "bounded_depth", | |
| "transformers_version": "5.10.2", | |
| "vocab_size": 50000 | |
| } | |