| { | |
| "dim": 5120, | |
| "n_layers": 48, | |
| "n_heads": 40, | |
| "n_kv_heads": 8, | |
| "head_dim": null, | |
| "vocab_size": -1, | |
| "multiple_of": 2048, | |
| "ffn_dim_multiplier": 1.2, | |
| "ffn_exp": 2.0, | |
| "norm_eps": 1e-05, | |
| "attention_chunk_size": null, | |
| "rope_theta": 500000.0, | |
| "use_scaled_rope": true, | |
| "rope_scaling_factor": 16, | |
| "rope_high_freq_factor": 1, | |
| "nope_layer_interval": null, | |
| "use_qk_norm": true, | |
| "attn_temperature_tuning": false, | |
| "floor_scale": 8192.0, | |
| "attn_scale": 0.1, | |
| "vision_args": { | |
| "image_size": { | |
| "height": 336, | |
| "width": 336 | |
| }, | |
| "patch_size": { | |
| "height": 14, | |
| "width": 14 | |
| }, | |
| "dim": 1408, | |
| "n_layers": 34, | |
| "n_heads": 16, | |
| "mlp_ratio": 4.0, | |
| "output_dim": 4096, | |
| "pixel_shuffle_ratio": 0.5 | |
| }, | |
| "moe_args": null, | |
| "quantization_args": null, | |
| "lora_args": null | |
| } | |