{
  "dim": 5120,
  "n_layers": 48,
  "n_heads": 40,
  "n_kv_heads": 8,
  "head_dim": null,
  "vocab_size": -1,
  "multiple_of": 2048,
  "ffn_dim_multiplier": 1.2,
  "ffn_exp": 2.0,
  "norm_eps": 1e-05,
  "attention_chunk_size": null,
  "rope_theta": 500000.0,
  "use_scaled_rope": true,
  "rope_scaling_factor": 16,
  "rope_high_freq_factor": 1,
  "nope_layer_interval": null,
  "use_qk_norm": true,
  "attn_temperature_tuning": false,
  "floor_scale": 8192.0,
  "attn_scale": 0.1,
  "vision_args": {
    "image_size": {
      "height": 336,
      "width": 336
    },
    "patch_size": {
      "height": 14,
      "width": 14
    },
    "dim": 1408,
    "n_layers": 34,
    "n_heads": 16,
    "mlp_ratio": 4.0,
    "output_dim": 4096,
    "pixel_shuffle_ratio": 0.5
  },
  "moe_args": null,
  "quantization_args": null,
  "lora_args": null
}