{
  "architectures": [
    "Dots3NoteForCausalLM"
  ],
  "apply_mla_qkv_lora_rescale": true,
  "attention_bias": false,
  "attention_dropout": 0.0,
  "attention_gate_type": "headwise",
  "bos_token_id": 151643,
  "eos_token_id": 151668,
  "first_k_dense_replace": 1,
  "hidden_act": "silu",
  "hidden_size": 5120,
  "index_head_dim": 128,
  "index_n_heads": 64,
  "index_topk": 2048,
  "initializer_range": 0.02,
  "intermediate_size": 13824,
  "kv_lora_rank": 512,
  "layer_types": [
    "full_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention"
  ],
  "max_position_embeddings": 524288,
  "model_type": "dots3_note",
  "moe_intermediate_size": 1536,
  "moe_layer_freq": 1,
  "n_routed_experts": 256,
  "n_shared_experts": 1,
  "norm_topk_prob": true,
  "num_attention_heads": 128,
  "num_experts_per_tok": 8,
  "num_hidden_layers": 46,
  "num_key_value_heads": 128,
  "pretraining_tp": 1,
  "q_lora_rank": 1024,
  "qk_nope_head_dim": 128,
  "qk_rope_head_dim": 64,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "rope_theta": 80000000.0,
  "routed_scaling_factor": 1.0,
  "scoring_func": "sigmoid",
  "sliding_window_size": 513,
  "swa_attention_gate_type": "headwise",
  "swa_kv_lora_rank": 1024,
  "swa_num_attention_heads": 64,
  "swa_num_key_value_heads": 64,
  "swa_q_lora_rank": 1024,
  "swa_qk_nope_head_dim": 192,
  "swa_qk_rope_head_dim": 64,
  "swa_rope_theta": 50000.0,
  "swa_v_head_dim": 128,
  "tie_word_embeddings": false,
  "topk_method": "noaux_tc",
  "torch_dtype": "bfloat16",
  "transformers_version": "4.36.0",
  "use_cache": true,
  "v_head_dim": 128,
  "vocab_size": 152064,
  "vision_config": {
    "embed_dim": 1536,
    "hidden_size": 5120,
    "intermediate_size": 4224,
    "moe_intermediate_size": 2112,
    "num_hidden_layers": 42,
    "num_attention_heads": 24,
    "num_channels": 3,
    "patch_size": 14,
    "spatial_merge_size": 2,
    "temporal_patch_size": 1,
    "rms_norm_eps": 1e-05,
    "use_bias": false,
    "use_qk_norm": true,
    "attn_implementation": "flash_attention_3",
    "initializer_range": 0.02,
    "is_causal": false,
    "post_norm": true,
    "pre_pixel_shuffle": true,
    "pyramid_num_routed": [
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      4,
      8,
      12,
      16,
      20,
      24,
      28,
      32,
      36,
      40,
      44,
      48,
      52,
      56,
      60,
      64,
      64
    ],
    "capacity_factor": 2,
    "router_scoring_func": "sigmoid",
    "router_scale": 1.0,
    "adapter_type": "patch_merger",
    "adapter_in_dim": 1536,
    "adapter_out_dim": 5120,
    "adapter_merge_size": 2
  },
  "audio_config": {
    "encoder_type": "dots",
    "whisper_config": {
      "d_model": 1280,
      "encoder_attention_heads": 20,
      "encoder_ffn_dim": 5120,
      "encoder_layers": 32,
      "num_mel_bins": 128,
      "max_source_positions": 6000,
      "activation_function": "swiglu"
    },
    "use_conv2d_stem": true,
    "use_rope": true,
    "use_rms_norm": true,
    "use_causal": false,
    "downsample_hidden_size": 480,
    "conv_chunksize": 500,
    "conv_stem_gradient_checkpointing": true,
    "conv_bucket_step": 10,
    "conv_bucket_max_elements": 20000,
    "rope_parameters": {
      "partial_rotary_factor": 0.5,
      "rope_theta": 10000.0
    },
    "merge_factor": 1,
    "chunk_seconds": 60,
    "whisper_adapter_in_dim": 1280,
    "whisper_adapter_out_dim": 5120,
    "sampling_rate": 16000,
    "audio_comp_start": "<|audio_comp_start|>",
    "audio_comp_span": "<|audio_comp_pad|>",
    "audio_comp_end": "<|audio_comp_end|>"
  }
}